do not autostart nse
[oweals/gnunet.git] / src / hostlist / hostlist-client.c
index 99b43f0082c8dd8e2701d521dc29f29fee8045dd..049e78c52f10131fc068d549bbb690550812478f 100644 (file)
@@ -4,7 +4,7 @@
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
@@ -22,6 +22,7 @@
  * @file hostlist/hostlist-client.c
  * @brief hostlist support.  Downloads HELLOs via HTTP.
  * @author Christian Grothoff
  * @file hostlist/hostlist-client.c
  * @brief hostlist support.  Downloads HELLOs via HTTP.
  * @author Christian Grothoff
+ * @author Matthias Wachs
  */
 
 #include "platform.h"
  */
 
 #include "platform.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet_transport_service.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet_transport_service.h"
+#include "gnunet-daemon-hostlist.h"
 #include <curl/curl.h>
 #include <curl/curl.h>
+#include "gnunet_common.h"
+#include "gnunet_bio_lib.h"
+
+#define DEBUG_HOSTLIST_CLIENT GNUNET_NO
 
 
-#define DEBUG_HOSTLIST_CLIENT GNUNET_YES
 
 /**
  * Number of connections that we must have to NOT download
 
 /**
  * Number of connections that we must have to NOT download
 #define MIN_CONNECTIONS 4
 
 /**
 #define MIN_CONNECTIONS 4
 
 /**
- * Our configuration.
+ * Interval between two advertised hostlist tests
  */
  */
-static const struct GNUNET_CONFIGURATION_Handle *cfg;
+#define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
+
+/**
+ * A single hostlist obtained by hostlist advertisements
+ */
+struct Hostlist
+{
+  /**
+   * previous entry, used to manage entries in a double linked list
+   */
+  struct Hostlist * prev;
+
+  /**
+   * next entry, used to manage entries in a double linked list
+   */
+  struct Hostlist * next;
+
+  /**
+   * URI where hostlist can be obtained
+   */
+  const char *hostlist_uri;
+
+  /**
+   * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
+   * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
+   * intial value = HOSTLIST_INITIAL
+   * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
+   * increased every successful download by number of obtained HELLO messages
+   * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
+   */
+  uint64_t quality;
+
+  /**
+   * Time the hostlist advertisement was recieved and the entry was created
+   */
+  struct GNUNET_TIME_Absolute time_creation;
+
+  /**
+   * Last time the hostlist was obtained
+   */
+  struct GNUNET_TIME_Absolute time_last_usage;
+
+  /**
+   * Number of HELLO messages obtained during last download
+   */
+  uint32_t hello_count;
+
+  /**
+   * Number of times the hostlist was successfully obtained
+   */
+  uint32_t times_used;
+
+};
+
 
 /**
 
 /**
- * Our scheduler.
+ * Our configuration.
  */
  */
-static struct GNUNET_SCHEDULER_Handle *sched;
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
 /**
  * Statistics handle.
  */
 
 /**
  * Statistics handle.
  */
-struct GNUNET_STATISTICS_Handle *stats; 
+static struct GNUNET_STATISTICS_Handle *stats; 
 
 /**
  * Transport handle.
  */
 
 /**
  * Transport handle.
  */
-struct GNUNET_TRANSPORT_Handle *transport;
+static struct GNUNET_TRANSPORT_Handle *transport;
                        
 /**
  * Proxy that we are using (can be NULL).
  */
 static char *proxy;
 
                        
 /**
  * Proxy that we are using (can be NULL).
  */
 static char *proxy;
 
-/**
- * Buffer for data downloaded via HTTP.
- */
-static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE];
-
 /**
  * Number of bytes valid in 'download_buffer'.
  */
 /**
  * Number of bytes valid in 'download_buffer'.
  */
@@ -91,30 +144,113 @@ static CURL *curl;
 static CURLM *multi;
 
 /**
 static CURLM *multi;
 
 /**
- * ID of the current task scheduled.
+ * How many bytes did we download from the current hostlist URL?
  */
  */
-static GNUNET_SCHEDULER_TaskIdentifier current_task;
-
+static uint32_t stat_bytes_downloaded;
 /**
  * Amount of time we wait between hostlist downloads.
  */
 static struct GNUNET_TIME_Relative hostlist_delay;
 
 /**
 /**
  * Amount of time we wait between hostlist downloads.
  */
 static struct GNUNET_TIME_Relative hostlist_delay;
 
 /**
- * Set to GNUNET_YES if the current URL had some problems.
- */ 
-static int bogus_url;
+ * ID of the task, checking if hostlist download should take plate
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;
 
 /**
 
 /**
- * Number of active connections (according to core service).
+ * ID of the task downloading the hostlist
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_download;
+
+/**
+ * ID of the task saving the hostlsit in a regular intervall
  */
  */
-static unsigned int connection_count;
+static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;
+
+/**
+ * ID of the task called to initiate a download
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;
+
+/**
+ * ID of the task controlling the locking between two hostlist tests
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;
 
 /**
  * At what time MUST the current hostlist request be done?
  */
 static struct GNUNET_TIME_Absolute end_time;
 
 
 /**
  * At what time MUST the current hostlist request be done?
  */
 static struct GNUNET_TIME_Absolute end_time;
 
+/**
+ * Head of the linked list used to store hostlists
+ */
+static struct Hostlist * linked_list_head;
+
+/**
+ *  Tail of the linked list used to store hostlists
+ */
+static struct Hostlist * linked_list_tail;
+
+/**
+ *  Current hostlist used for downloading
+ */
+static struct Hostlist * current_hostlist;
+
+/**
+ *  Size of the linke list  used to store hostlists
+ */
+static unsigned int linked_list_size;
+
+/**
+ * Head of the linked list used to store hostlists
+ */
+static struct Hostlist * hostlist_to_test;
+
+/**
+ * Set to GNUNET_YES if the current URL had some problems.
+ */
+static int stat_bogus_url;
+
+/**
+ * Value controlling if a hostlist is tested at the moment
+ */
+static int stat_testing_hostlist;
+
+/**
+ * Value controlling if a hostlist testing is allowed at the moment
+ */
+static int stat_testing_allowed;
+
+/**
+ * Value controlling if a hostlist download is running at the moment
+ */
+static int stat_download_in_progress;
+
+/**
+ * Value saying if a preconfigured bootstrap server is used
+ */
+static unsigned int stat_use_bootstrap;
+/**
+ * Set if we are allowed to learn new hostlists and use them
+ */
+static int stat_learning;
+
+/**
+ * Value saying if hostlist download was successful
+ */
+static unsigned int stat_download_successful;
+
+/**
+ * Value saying how many valid HELLO messages were obtained during download
+ */
+static unsigned int stat_hellos_obtained;
+
+/**
+ * Number of active connections (according to core service).
+ */
+static unsigned int stat_connection_count;
+
 
 /**
  * Process downloaded bits by calling callback on each HELLO.
 
 /**
  * Process downloaded bits by calling callback on each HELLO.
@@ -126,11 +262,12 @@ static struct GNUNET_TIME_Absolute end_time;
  * @return number of bytes that were processed (always size*nmemb)
  */
 static size_t
  * @return number of bytes that were processed (always size*nmemb)
  */
 static size_t
-download_hostlist_processor (void *ptr, 
+callback_download (void *ptr, 
                             size_t size, 
                             size_t nmemb, 
                             void *ctx)
 {
                             size_t size, 
                             size_t nmemb, 
                             void *ctx)
 {
+  static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
   const char * cbuf = ptr;
   const struct GNUNET_MessageHeader *msg;
   size_t total;
   const char * cbuf = ptr;
   const struct GNUNET_MessageHeader *msg;
   size_t total;
@@ -139,22 +276,24 @@ download_hostlist_processor (void *ptr,
   uint16_t msize;
 
   total = size * nmemb;
   uint16_t msize;
 
   total = size * nmemb;
-  if ( (total == 0) || (bogus_url) )
+  stat_bytes_downloaded += total;
+  if ( (total == 0) || (stat_bogus_url) )
     {
       return total;  /* ok, no data or bogus data */
     }
     {
       return total;  /* ok, no data or bogus data */
     }
+
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# bytes downloaded from hostlist servers"), 
                            (int64_t) total, 
                            GNUNET_NO);  
   left = total;
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# bytes downloaded from hostlist servers"), 
                            (int64_t) total, 
                            GNUNET_NO);  
   left = total;
-  while (left > 0)
+  while ( (left > 0) ||
+         (download_pos > 0) )
     {
     {
-      cpy = GNUNET_MIN (total, GNUNET_SERVER_MAX_MESSAGE_SIZE - download_pos);
-      GNUNET_assert (cpy > 0);
+      cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
       memcpy (&download_buffer[download_pos],
              cbuf,
       memcpy (&download_buffer[download_pos],
              cbuf,
-             cpy);
+             cpy);      
       cbuf += cpy;
       download_pos += cpy;
       left -= cpy;
       cbuf += cpy;
       download_pos += cpy;
       left -= cpy;
@@ -174,8 +313,9 @@ download_hostlist_processor (void *ptr,
          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                      _("Invalid `%s' message received from hostlist at `%s'\n"),
                      "HELLO",
          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                      _("Invalid `%s' message received from hostlist at `%s'\n"),
                      "HELLO",
-                     current_url); 
-         bogus_url = 1;
+                     current_url);
+          stat_hellos_obtained++;
+         stat_bogus_url = 1;
          return total;
        }
       if (download_pos < msize)
          return total;
        }
       if (download_pos < msize)
@@ -193,8 +333,9 @@ download_hostlist_processor (void *ptr,
          GNUNET_STATISTICS_update (stats, 
                                    gettext_noop ("# valid HELLOs downloaded from hostlist servers"), 
                                    1, 
          GNUNET_STATISTICS_update (stats, 
                                    gettext_noop ("# valid HELLOs downloaded from hostlist servers"), 
                                    1, 
-                                   GNUNET_NO);  
-         GNUNET_TRANSPORT_offer_hello (transport, msg);
+                                   GNUNET_NO);
+         stat_hellos_obtained++;
+         GNUNET_TRANSPORT_offer_hello (transport, msg, NULL, NULL);
        }
       else
        {
        }
       else
        {
@@ -206,7 +347,8 @@ download_hostlist_processor (void *ptr,
                      _("Invalid `%s' message received from hostlist at `%s'\n"),
                      "HELLO",
                      current_url);
                      _("Invalid `%s' message received from hostlist at `%s'\n"),
                      "HELLO",
                      current_url);
-         bogus_url = GNUNET_YES;
+         stat_bogus_url = GNUNET_YES;
+          stat_hellos_obtained++;
          return total;
        }
       memmove (download_buffer,
          return total;
        }
       memmove (download_buffer,
@@ -224,7 +366,7 @@ download_hostlist_processor (void *ptr,
  * @return NULL if there is no URL available
  */
 static char *
  * @return NULL if there is no URL available
  */
 static char *
-get_url ()
+get_bootstrap_server ()
 {
   char *servers;
   char *ret;
 {
   char *servers;
   char *ret;
@@ -233,13 +375,13 @@ get_url ()
 
   if (GNUNET_OK != 
       GNUNET_CONFIGURATION_get_value_string (cfg,
 
   if (GNUNET_OK != 
       GNUNET_CONFIGURATION_get_value_string (cfg,
-                                            "HOSTLIST",
-                                            "SERVERS",
-                                            &servers))
+                                             "HOSTLIST",
+                                             "SERVERS",
+                                             &servers))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
-                 "SERVERS", "HOSTLIST");
+                  _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
+                  "SERVERS", "HOSTLIST");
       return NULL;
     }
 
       return NULL;
     }
 
@@ -258,8 +400,8 @@ get_url ()
   if (urls == 0)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
   if (urls == 0)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
-                 "SERVERS", "HOSTLIST");
+                  _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
+                  "SERVERS", "HOSTLIST");
       GNUNET_free (servers);
       return NULL;
     }
       GNUNET_free (servers);
       return NULL;
     }
@@ -285,17 +427,224 @@ get_url ()
   return ret;
 }
 
   return ret;
 }
 
+/**
+ * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
+ * @return uri to use, NULL if there is no URL available
+ */
+static char *
+download_get_url ()
+{
+  uint32_t index;
+  unsigned int counter;
+  struct Hostlist * pos;
+
+  if ( GNUNET_NO == stat_learning)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Using preconfigured bootstrap server\n");
+    current_hostlist = NULL;
+    return get_bootstrap_server();
+  }
+
+  if ( ( GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Testing new advertised hostlist if it is obtainable\n");
+    current_hostlist = hostlist_to_test;
+    return strdup(hostlist_to_test->hostlist_uri);
+  }
+
+  if ( (GNUNET_YES == stat_use_bootstrap) ||
+       (linked_list_size == 0) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Using preconfigured bootstrap server\n");
+    current_hostlist = NULL;
+    return get_bootstrap_server();
+  }
+  index = GNUNET_CRYPTO_random_u32 ( GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
+  counter = 0;
+  pos = linked_list_head;
+  while ( counter < index )
+    {
+      pos = pos->next;
+      counter ++;
+    }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Using learned hostlist `%s'\n", pos->hostlist_uri);
+  current_hostlist = pos;
+  return strdup(pos->hostlist_uri);
+}
+
 
 
-#define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt(c, a, b); if (ret != CURLE_OK) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror(ret)); } while (0);
+#define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt(c, a, b); if (ret != CURLE_OK) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror(ret)); } while (0)
 
 
+/**
+ * Method to save hostlist to a file during hostlist client shutdown
+ * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
+ */
+static void save_hostlist_file ( int shutdown );
 
 /**
 
 /**
- * Schedule the background task that will (possibly)
- * download a hostlist.
+ * add val2 to val1 with overflow check
+ * @param val1 value 1
+ * @param val2 value 2
+ * @return result
+ */
+static uint64_t checked_add (uint64_t val1, uint64_t val2)
+{
+  static uint64_t temp;
+  static uint64_t maxv;
+
+  maxv = 0;
+  maxv--;
+
+  temp = val1+val2;
+  if ( temp < val1)
+    return maxv;
+  else
+    return temp;
+}
+
+/**
+ * Subtract val2 from val1 with underflow check
+ * @param val1 value 1
+ * @param val2 value 2
+ * @return result
+ */
+static uint64_t checked_sub (uint64_t val1, uint64_t val2)
+{
+  if ( val1 <= val2)
+    return 0;
+  else
+    return (val1-val2);
+}
+
+/**
+ * Method to check if  a URI is in hostlist linked list
+ * @param uri uri to check
+ * @return GNUNET_YES if existing in linked list, GNUNET_NO if not
+ */
+static int
+linked_list_contains (const char * uri)
+{
+  struct Hostlist * pos;
+
+  pos = linked_list_head;
+  while (pos != NULL)
+    {
+      if (0 == strcmp(pos->hostlist_uri, uri) )
+        return GNUNET_YES;
+      pos = pos->next;
+    }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Method returning the hostlist element with the lowest quality in the datastore
+ * @return hostlist with lowest quality
+ */
+static struct Hostlist *
+linked_list_get_lowest_quality ( )
+{
+  struct Hostlist * pos;
+  struct Hostlist * lowest;
+
+  if (linked_list_size == 0)
+    return NULL;
+  lowest = linked_list_head;
+  pos = linked_list_head->next;
+  while (pos != NULL)
+    {
+      if (pos->quality < lowest->quality)
+        lowest = pos;
+      pos = pos->next;
+    }
+  return lowest;
+}
+
+
+/**
+ * Method to insert a hostlist into the datastore. If datastore contains maximum number of elements, the elements with lowest quality is dismissed
  */
 static void
  */
 static void
-schedule_hostlist_task (void);
+insert_hostlist ( )
+{
+  struct Hostlist * lowest_quality;
+
+  if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
+    {
+      /* No free entries available, replace existing entry  */
+      lowest_quality = linked_list_get_lowest_quality();
+      GNUNET_assert (lowest_quality != NULL);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
+                 lowest_quality->hostlist_uri,
+                 (unsigned long long) lowest_quality->quality);
+      GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, lowest_quality);
+      linked_list_size--;
+      GNUNET_free (lowest_quality);
+    }
+  GNUNET_CONTAINER_DLL_insert(linked_list_head,
+                             linked_list_tail,
+                             hostlist_to_test);
+  linked_list_size++;
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop("# advertised hostlist URIs"),
+                         linked_list_size,
+                         GNUNET_NO);
+  stat_testing_hostlist = GNUNET_NO;
+}
+
+
+/**
+ * Method updating hostlist statistics
+ */
+static void update_hostlist ( )
+{
+  char *stat;
+  if ( ((stat_use_bootstrap == GNUNET_NO) && ( NULL != current_hostlist )) ||
+       ((stat_testing_hostlist == GNUNET_YES) && ( NULL != current_hostlist )) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Updating hostlist statics for URI `%s'\n",current_hostlist->hostlist_uri );
+     current_hostlist->hello_count = stat_hellos_obtained;
+     current_hostlist->time_last_usage = GNUNET_TIME_absolute_get();
+     current_hostlist->quality = checked_add ( current_hostlist->quality, (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
+     if ( GNUNET_YES == stat_download_successful )
+     {
+       current_hostlist->times_used++;
+       current_hostlist->quality = checked_add ( current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
+       GNUNET_asprintf (&stat,
+                        gettext_noop("# advertised URI `%s' downloaded"),
+                        current_hostlist->hostlist_uri);
+
+       GNUNET_STATISTICS_update ( stats,
+                                  stat,
+                                  1,
+                                  GNUNET_YES);
+       GNUNET_free (stat);
+     }
+     else
+       current_hostlist->quality = checked_sub ( current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD );
+  }
+  current_hostlist = NULL;
+  /* Alternating the usage of preconfigured and learned hostlists */
+
+  if (stat_testing_hostlist == GNUNET_YES)
+    return;
 
 
+  if ( GNUNET_YES == stat_learning)
+    {
+    if (stat_use_bootstrap == GNUNET_YES)
+      stat_use_bootstrap = GNUNET_NO;
+    else
+      stat_use_bootstrap = GNUNET_YES;
+    }
+  else
+    stat_use_bootstrap = GNUNET_YES;
+}
 
 /**
  * Clean up the state from the task that downloaded the
 
 /**
  * Clean up the state from the task that downloaded the
@@ -306,6 +655,22 @@ clean_up ()
 {
   CURLMcode mret;
 
 {
   CURLMcode mret;
 
+  if ( ( stat_testing_hostlist == GNUNET_YES ) && ( GNUNET_NO == stat_download_successful) && (NULL != hostlist_to_test))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),hostlist_to_test->hostlist_uri);
+  }
+
+  if ( stat_testing_hostlist == GNUNET_YES )
+    {
+    stat_testing_hostlist = GNUNET_NO;
+    }
+  if ( NULL != hostlist_to_test)
+  {
+    GNUNET_free (hostlist_to_test);
+    hostlist_to_test = NULL;
+  }
+
   if (multi != NULL)
     {
       mret = curl_multi_remove_handle (multi, curl);
   if (multi != NULL)
     {
       mret = curl_multi_remove_handle (multi, curl);
@@ -331,16 +696,84 @@ clean_up ()
     }  
   GNUNET_free_non_null (current_url);
   current_url = NULL;
     }  
   GNUNET_free_non_null (current_url);
   current_url = NULL;
-  schedule_hostlist_task ();
+  stat_bytes_downloaded = 0;
+  stat_download_in_progress = GNUNET_NO;
 }
 
 }
 
+/**
+ * Task that is run when we are ready to receive more data from the hostlist
+ * server.
+ *
+ * @param cls closure, unused
+ * @param tc task context, unused
+ */
+static void
+task_download (void *cls,
+             const struct GNUNET_SCHEDULER_TaskContext *tc);
 
 /**
  * Ask CURL for the select set and then schedule the
  * receiving task with the scheduler.
  */
 static void
 
 /**
  * Ask CURL for the select set and then schedule the
  * receiving task with the scheduler.
  */
 static void
-run_multi ();
+download_prepare ()
+{
+  CURLMcode mret;
+  fd_set rs;
+  fd_set ws;
+  fd_set es;
+  int max;
+  struct GNUNET_NETWORK_FDSet *grs;
+  struct GNUNET_NETWORK_FDSet *gws;
+  long timeout;
+  struct GNUNET_TIME_Relative rtime;
+
+  max = -1;
+  FD_ZERO (&rs);
+  FD_ZERO (&ws);
+  FD_ZERO (&es);
+  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
+  if (mret != CURLM_OK)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("%s failed at %s:%d: `%s'\n"),
+                  "curl_multi_fdset", __FILE__, __LINE__,
+                  curl_multi_strerror (mret));
+      clean_up ();
+      return;
+    }
+  mret = curl_multi_timeout (multi, &timeout);
+  if (mret != CURLM_OK)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("%s failed at %s:%d: `%s'\n"),
+                  "curl_multi_timeout", __FILE__, __LINE__,
+                  curl_multi_strerror (mret));
+      clean_up ();
+      return;
+    }
+  rtime = GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
+                                    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
+                                                                   timeout));
+  grs = GNUNET_NETWORK_fdset_create ();
+  gws = GNUNET_NETWORK_fdset_create ();
+  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
+  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
+#if DEBUG_HOSTLIST_CLIENT
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Scheduling task for hostlist download using cURL\n");
+#endif
+  ti_download
+    = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
+                                   GNUNET_SCHEDULER_NO_TASK,
+                                   rtime,
+                                   grs,
+                                   gws,
+                                   &task_download,
+                                   multi);
+  GNUNET_NETWORK_fdset_destroy (gws);
+  GNUNET_NETWORK_fdset_destroy (grs);
+}
 
 
 /**
 
 
 /**
@@ -351,13 +784,15 @@ run_multi ();
  * @param tc task context, unused
  */
 static void
  * @param tc task context, unused
  */
 static void
-multi_ready (void *cls,
+task_download (void *cls,
             const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
             const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
+
   int running;
   struct CURLMsg *msg;
   CURLMcode mret;
   
   int running;
   struct CURLMsg *msg;
   CURLMcode mret;
   
+  ti_download = GNUNET_SCHEDULER_NO_TASK;
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     {
 #if DEBUG_HOSTLIST_CLIENT
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     {
 #if DEBUG_HOSTLIST_CLIENT
@@ -365,14 +800,16 @@ multi_ready (void *cls,
                  "Shutdown requested while trying to download hostlist from `%s'\n",
                  current_url);
 #endif
                  "Shutdown requested while trying to download hostlist from `%s'\n",
                  current_url);
 #endif
+      update_hostlist();
       clean_up ();
       return;
     }
       clean_up ();
       return;
     }
-  if (GNUNET_TIME_absolute_get_remaining (end_time).value == 0)
+  if (GNUNET_TIME_absolute_get_remaining (end_time).rel_value == 0)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("Timeout trying to download hostlist from `%s'\n"),
                  current_url);
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("Timeout trying to download hostlist from `%s'\n"),
                  current_url);
+      update_hostlist();
       clean_up ();
       return;
     }
       clean_up ();
       return;
     }
@@ -380,14 +817,24 @@ multi_ready (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Ready for processing hostlist client request\n");
 #endif
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Ready for processing hostlist client request\n");
 #endif
+
   do 
     {
       running = 0;
   do 
     {
       running = 0;
+      if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
+        {
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                                  _("Download limit of %u bytes exceeded, stopping download\n"),MAX_BYTES_PER_HOSTLISTS);
+        clean_up();
+        return;
+        }
       mret = curl_multi_perform (multi, &running);
       if (running == 0)
        {
          do
            {
       mret = curl_multi_perform (multi, &running);
       if (running == 0)
        {
          do
            {
+
+
              msg = curl_multi_info_read (multi, &running);
              GNUNET_break (msg != NULL);
              if (msg == NULL)
              msg = curl_multi_info_read (multi, &running);
              GNUNET_break (msg != NULL);
              if (msg == NULL)
@@ -405,19 +852,33 @@ multi_ready (void *cls,
                               __LINE__,
                               curl_easy_strerror (msg->data.result));            
                  else
                               __LINE__,
                               curl_easy_strerror (msg->data.result));            
                  else
+                   {
                    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                                _("Download of hostlist `%s' completed.\n"),
                                current_url);
                    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                                _("Download of hostlist `%s' completed.\n"),
                                current_url);
+                   stat_download_successful = GNUNET_YES;
+                   update_hostlist();
+                   if (GNUNET_YES == stat_testing_hostlist)
+                    {
+                      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                                        _("Adding successfully tested hostlist `%s' datastore.\n"),current_url);
+                     insert_hostlist();
+                     hostlist_to_test = NULL;
+                     stat_testing_hostlist = GNUNET_NO;
+                    }
+                   }
                  clean_up ();
                  return;
                default:
                  break;
                }
                  clean_up ();
                  return;
                default:
                  break;
                }
+
            }
            }
-         while (running > 0);
+         while ( (running > 0) );
        }
     }
   while (mret == CURLM_CALL_MULTI_PERFORM);
        }
     }
   while (mret == CURLM_CALL_MULTI_PERFORM);
+
   if (mret != CURLM_OK)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
   if (mret != CURLM_OK)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -426,73 +887,7 @@ multi_ready (void *cls,
                  curl_multi_strerror (mret));
       clean_up ();
     }
                  curl_multi_strerror (mret));
       clean_up ();
     }
-  run_multi ();
-}
-
-
-/**
- * Ask CURL for the select set and then schedule the
- * receiving task with the scheduler.
- */
-static void
-run_multi () 
-{
-  CURLMcode mret;
-  fd_set rs;
-  fd_set ws;
-  fd_set es;
-  int max;
-  struct GNUNET_NETWORK_FDSet *grs;
-  struct GNUNET_NETWORK_FDSet *gws;
-  long timeout;
-  struct GNUNET_TIME_Relative rtime;
-  
-  max = -1;
-  FD_ZERO (&rs);
-  FD_ZERO (&ws);
-  FD_ZERO (&es);
-  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
-  if (mret != CURLM_OK)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("%s failed at %s:%d: `%s'\n"),
-                 "curl_multi_fdset", __FILE__, __LINE__,
-                 curl_multi_strerror (mret));
-      clean_up ();
-      return;
-    }
-  mret = curl_multi_timeout (multi, &timeout);
-  if (mret != CURLM_OK)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("%s failed at %s:%d: `%s'\n"),
-                 "curl_multi_timeout", __FILE__, __LINE__,
-                 curl_multi_strerror (mret));
-      clean_up ();
-      return;
-    }
-  rtime = GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
-                                   GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                                                  timeout));
-  grs = GNUNET_NETWORK_fdset_create ();
-  gws = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
-  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);  
-#if DEBUG_HOSTLIST_CLIENT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Scheduling task for hostlist download using cURL\n");
-#endif
-  current_task 
-    = GNUNET_SCHEDULER_add_select (sched,
-                                  GNUNET_SCHEDULER_PRIORITY_DEFAULT,
-                                  GNUNET_SCHEDULER_NO_TASK,
-                                  rtime,
-                                  grs,
-                                  gws,
-                                  &multi_ready,
-                                  multi);
-  GNUNET_NETWORK_fdset_destroy (gws);
-  GNUNET_NETWORK_fdset_destroy (grs);
+  download_prepare ();
 }
 
 
 }
 
 
@@ -506,6 +901,10 @@ download_hostlist ()
   CURLcode ret;
   CURLMcode mret;
 
   CURLcode ret;
   CURLMcode mret;
 
+
+  current_url = download_get_url ();
+  if (current_url == NULL)
+    return;
   curl = curl_easy_init ();
   multi = NULL;
   if (curl == NULL)
   curl = curl_easy_init ();
   multi = NULL;
   if (curl == NULL)
@@ -514,10 +913,15 @@ download_hostlist ()
       clean_up ();
       return;
     }
       clean_up ();
       return;
     }
-  current_url = get_url ();
   GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
              _("Bootstrapping using hostlist at `%s'.\n"), 
              current_url);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
              _("Bootstrapping using hostlist at `%s'.\n"), 
              current_url);
+
+  stat_download_in_progress = GNUNET_YES;
+  stat_download_successful = GNUNET_NO;
+  stat_hellos_obtained = 0;
+  stat_bytes_downloaded = 0;
+
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# hostlist downloads initiated"), 
                            1, 
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# hostlist downloads initiated"), 
                            1, 
@@ -525,10 +929,10 @@ download_hostlist ()
   if (proxy != NULL)
     CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);    
   download_pos = 0;
   if (proxy != NULL)
     CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);    
   download_pos = 0;
-  bogus_url = 0;
+  stat_bogus_url = 0;
   CURL_EASY_SETOPT (curl,
                    CURLOPT_WRITEFUNCTION, 
   CURL_EASY_SETOPT (curl,
                    CURLOPT_WRITEFUNCTION, 
-                   &download_hostlist_processor);
+                   &callback_download);
   if (ret != CURLE_OK)
     {
       clean_up ();
   if (ret != CURLE_OK)
     {
       clean_up ();
@@ -582,7 +986,7 @@ download_hostlist ()
   if (multi == NULL)
     {
       GNUNET_break (0);
   if (multi == NULL)
     {
       GNUNET_break (0);
-      clean_up ();
+      /* clean_up (); */
       return;
     }
   mret = curl_multi_add_handle (multi, curl);
       return;
     }
   mret = curl_multi_add_handle (multi, curl);
@@ -603,115 +1007,260 @@ download_hostlist ()
       return;
     }
   end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
       return;
     }
   end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
-  run_multi ();
+  download_prepare ();
 }  
 
 
 }  
 
 
+static void
+task_download_dispatcher (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
+    if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+      return;
+   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Download is initiated...\n");
+   if ( GNUNET_NO == stat_download_in_progress )
+   {
+     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+               "Download can start immediately...\n");
+     download_hostlist();
+   }
+   else
+   {
+     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+               "Download in progess, have to wait...\n");
+     ti_download_dispatcher_task = GNUNET_SCHEDULER_add_delayed (WAITING_INTERVALL,
+                                                              &task_download_dispatcher,
+                                                              NULL);
+   }
+}
+
 /**
  * Task that checks if we should try to download a hostlist.
  * If so, we initiate the download, otherwise we schedule
  * this task again for a later time.
  */
 static void
 /**
  * Task that checks if we should try to download a hostlist.
  * If so, we initiate the download, otherwise we schedule
  * this task again for a later time.
  */
 static void
-check_task (void *cls,
+task_check (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
            const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  current_task = GNUNET_SCHEDULER_NO_TASK;
+  static int once;
+  struct GNUNET_TIME_Relative delay;
+
+  ti_check_download = GNUNET_SCHEDULER_NO_TASK;
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     return;
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     return;
-  if (connection_count < MIN_CONNECTIONS)
-    download_hostlist ();
-  else
-    schedule_hostlist_task ();
-}
-
 
 
-/**
- * Compute when we should check the next time about downloading
- * a hostlist; then schedule the task accordingly.
- */
-static void
-schedule_hostlist_task ()
-{
-  static int once;
-  struct GNUNET_TIME_Relative delay;
+  if (stat_connection_count < MIN_CONNECTIONS)
+  {
+    ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
+                                                          NULL);
+  }
 
   if (stats == NULL)
 
   if (stats == NULL)
-    {
-      curl_global_cleanup ();
-      return; /* in shutdown */
-    }
+  {
+    curl_global_cleanup ();
+    return; /* in shutdown */
+  }
   delay = hostlist_delay;
   delay = hostlist_delay;
-  if (hostlist_delay.value == 0)
+  if (hostlist_delay.rel_value == 0)
     hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
   else
     hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
     hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
   else
     hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
-  if (hostlist_delay.value > GNUNET_TIME_UNIT_HOURS.value * (1 + connection_count))
+  if (hostlist_delay.rel_value > GNUNET_TIME_UNIT_HOURS.rel_value * (1 + stat_connection_count))
     hostlist_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
     hostlist_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
-                                                   (1 + connection_count));
+                                                    (1 + stat_connection_count));
   GNUNET_STATISTICS_set (stats,
   GNUNET_STATISTICS_set (stats,
-                        gettext_noop("Minimum time between hostlist downloads"),
-                        hostlist_delay.value,
-                        GNUNET_YES);
+                         gettext_noop("# milliseconds between hostlist downloads"),
+                         hostlist_delay.rel_value,
+                         GNUNET_YES);
   if (0 == once)
     {
       delay = GNUNET_TIME_UNIT_ZERO;
       once = 1;
     }  
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
   if (0 == once)
     {
       delay = GNUNET_TIME_UNIT_ZERO;
       once = 1;
     }  
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-             _("Have %u/%u connections.  Will consider downloading hostlist in %llums\n"),
-             connection_count,
-             MIN_CONNECTIONS,
-             (unsigned long long) delay.value);
-  current_task = GNUNET_SCHEDULER_add_delayed (sched,
-                                              delay,
-                                              &check_task,
-                                              NULL);
+              _("Have %u/%u connections.  Will consider downloading hostlist in %llums\n"),
+              stat_connection_count,
+              MIN_CONNECTIONS,
+              (unsigned long long) delay.rel_value);
+  ti_check_download = GNUNET_SCHEDULER_add_delayed (delay,
+                                               &task_check,
+                                               NULL);
+}
+
+/**
+ * This tasks sets hostlist testing to allowed after intervall between to testings is reached
+ * cls closure
+ * tc TaskContext
+ */
+static void
+task_testing_intervall_reset (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+   stat_testing_allowed = GNUNET_OK;
+   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Testing new hostlist advertisements is allowed again\n");
 }
 
 
 }
 
 
+/**
+ * Task that writes hostlist entries to a file on a regular base
+ * cls closure
+ * tc TaskContext
+ */
+static void
+task_hostlist_saving (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Scheduled saving of hostlists\n"));
+  save_hostlist_file ( GNUNET_NO );
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Hostlists will be saved to file again in %llums\n"),
+              (unsigned long long) SAVING_INTERVALL.rel_value);
+  ti_saving_task = GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL,
+                                               &task_hostlist_saving,
+                                               NULL);
+}
+
 /**
  * Method called whenever a given peer connects.
  *
  * @param cls closure
  * @param peer peer identity this notification is about
 /**
  * Method called whenever a given peer connects.
  *
  * @param cls closure
  * @param peer peer identity this notification is about
- * @param latency reported latency of the connection with 'other'
- * @param distance reported distance (DV) to 'other' 
+ * @param atsi performance data
  */
 static void
  */
 static void
-connect_handler (void *cls,
+handler_connect (void *cls,
                 const struct
                 GNUNET_PeerIdentity * peer,
                 const struct
                 GNUNET_PeerIdentity * peer,
-                struct GNUNET_TIME_Relative latency,
-                uint32_t distance)
+                const struct GNUNET_TRANSPORT_ATS_Information *atsi)
 {
 {
-  connection_count++;
+  GNUNET_assert (stat_connection_count < UINT_MAX);
+  stat_connection_count++;
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# active connections"), 
                            1, 
   GNUNET_STATISTICS_update (stats, 
                            gettext_noop ("# active connections"), 
                            1, 
-                           GNUNET_NO);  
+                           GNUNET_NO);
 }
 
 
 /**
 }
 
 
 /**
- * Method called whenever a given peer connects.
+ * Method called whenever a given peer disconnects.
  *
  * @param cls closure
  * @param peer peer identity this notification is about
  */
 static void
  *
  * @param cls closure
  * @param peer peer identity this notification is about
  */
 static void
-disconnect_handler (void *cls,
+handler_disconnect (void *cls,
                    const struct
                    GNUNET_PeerIdentity * peer)
 {
                    const struct
                    GNUNET_PeerIdentity * peer)
 {
-  connection_count--;
-  GNUNET_STATISTICS_update (stats, 
-                           gettext_noop ("# active connections"), 
-                           -1, 
-                           GNUNET_NO);  
+  GNUNET_assert (stat_connection_count > 0);
+  stat_connection_count--;
+  GNUNET_STATISTICS_update (stats,
+                           gettext_noop ("# active connections"),
+                           -1,
+                           GNUNET_NO);
+}
+
+/**
+ * Method called whenever an advertisement message arrives.
+ *
+ * @param cls closure (always NULL)
+ * @param peer the peer sending the message
+ * @param message the actual message
+ * @param atsi performance data
+ * @return GNUNET_OK to keep the connection open,
+ *         GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+handler_advertisement (void *cls,
+                      const struct GNUNET_PeerIdentity * peer,
+                      const struct GNUNET_MessageHeader * message,
+                      const struct GNUNET_TRANSPORT_ATS_Information *atsi)
+{
+  size_t size;
+  size_t uri_size;
+  const struct GNUNET_MessageHeader * incoming;
+  const char *uri;
+  struct Hostlist * hostlist;
+
+  GNUNET_assert (ntohs (message->type) == GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
+  size = ntohs (message->size);
+  if (size <= sizeof(struct GNUNET_MessageHeader))
+    {
+      GNUNET_break_op (0);
+      return GNUNET_SYSERR;
+    }
+  incoming = (const struct GNUNET_MessageHeader *) message;
+  uri = (const char*) &incoming[1];
+  uri_size = size - sizeof (struct GNUNET_MessageHeader);
+  if (uri [uri_size - 1] != '\0')
+    {
+      GNUNET_break_op (0);
+      return GNUNET_SYSERR;
+    }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hostlist client recieved advertisement from `%s' containing URI `%s'\n", 
+             GNUNET_i2s (peer), 
+             uri);
+  if (GNUNET_NO != linked_list_contains (uri))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "URI `%s' is already known\n",
+                uri);
+      return GNUNET_OK;
+    }
+
+  if ( GNUNET_NO == stat_testing_allowed )
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
+      return GNUNET_SYSERR;
+    }
+  if ( GNUNET_YES == stat_testing_hostlist )
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Currently not accepting new advertisements: we are already testing a hostlist\n");
+      return GNUNET_SYSERR;
+    }
+
+  hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
+  hostlist->hostlist_uri = (const char*) &hostlist[1];
+  memcpy (&hostlist[1], uri, uri_size);
+  hostlist->time_creation = GNUNET_TIME_absolute_get();
+  hostlist->time_last_usage = GNUNET_TIME_absolute_get_zero();
+  hostlist->quality = HOSTLIST_INITIAL;
+  hostlist_to_test = hostlist;
+
+  stat_testing_hostlist = GNUNET_YES;
+  stat_testing_allowed = GNUNET_NO;
+  ti_testing_intervall_task = GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
+                                                         &task_testing_intervall_reset,
+                                                         NULL);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+            "Testing new hostlist advertisements is locked for the next %u ms\n",
+            TESTING_INTERVAL.rel_value);
+
+  ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
+                                                     NULL);
+
+  return GNUNET_OK;
 }
 
 
 }
 
 
+
 /**
  * Continuation called by the statistics code once 
  * we go the stat.  Initiates hostlist download scheduling.
 /**
  * Continuation called by the statistics code once 
  * we go the stat.  Initiates hostlist download scheduling.
@@ -729,7 +1278,8 @@ primary_task (void *cls, int success)
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Statistics request done, scheduling hostlist download\n");
 #endif
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Statistics request done, scheduling hostlist download\n");
 #endif
-  schedule_hostlist_task ();
+  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
+                                           NULL);
 }
 
 
 }
 
 
@@ -743,34 +1293,223 @@ process_stat (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Initial time between hostlist downloads is %llums\n"),
              (unsigned long long) value);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Initial time between hostlist downloads is %llums\n"),
              (unsigned long long) value);
-  hostlist_delay.value = value;
+  hostlist_delay.rel_value = value;
   return GNUNET_OK;
 }
 
   return GNUNET_OK;
 }
 
+/**
+ * Method to load persistent hostlist file during hostlist client startup
+ */
+static void 
+load_hostlist_file ()
+{
+  char *filename;
+  char *uri;
+  char *emsg;
+  struct Hostlist * hostlist;
+  uri = NULL;
+  uint32_t times_used;
+  uint32_t hellos_returned;
+  uint64_t quality;
+  uint64_t last_used;
+  uint64_t created;
+  uint32_t counter;
+
+  if (GNUNET_OK !=
+                 GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                                   "HOSTLIST",
+                                                   "HOSTLISTFILE",
+                                                   &filename))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("No `%s' specified in `%s' configuration, cannot load hostlists from file.\n"),
+                  "HOSTLISTFILE", "HOSTLIST");
+      return;
+    }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading saved hostlist entries from file `%s' \n"), filename);
+  if ( GNUNET_NO == GNUNET_DISK_file_test (filename) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Hostlist file `%s' is not existing\n"), filename);
+    GNUNET_free ( filename );
+    return;
+  }
+
+  struct GNUNET_BIO_ReadHandle * rh = GNUNET_BIO_read_open (filename);
+  if (NULL == rh)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  _("Could not open file `%s' for reading to load hostlists: %s\n"), 
+                 filename,
+                 STRERROR (errno));
+      GNUNET_free (filename);
+      return;
+    }
+
+  counter = 0;
+  while ( (GNUNET_OK == GNUNET_BIO_read_string (rh, "url" , &uri, MAX_URL_LEN)) &&
+         (NULL != uri) &&
+         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)) )
+    {
+      hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
+      hostlist->hello_count = hellos_returned;
+      hostlist->hostlist_uri = (const char *) &hostlist[1];
+      memcpy (&hostlist[1], uri, strlen(uri)+1);
+      hostlist->quality = quality;
+      hostlist->time_creation.abs_value = created;
+      hostlist->time_last_usage.abs_value = last_used;
+      GNUNET_CONTAINER_DLL_insert(linked_list_head, linked_list_tail, hostlist);
+      linked_list_size++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Added hostlist entry eith URI `%s' \n", hostlist->hostlist_uri);
+      GNUNET_free (uri);
+      uri = NULL;
+      counter++;
+      if ( counter >= MAX_NUMBER_HOSTLISTS ) break;
+    }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("%u hostlist URIs loaded from file\n"), counter);
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop("# hostlist URIs read from file"),
+                         counter,
+                         GNUNET_YES);
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop("# advertised hostlist URIs"),
+                         linked_list_size,
+                         GNUNET_NO);
+
+  GNUNET_free_non_null (uri);
+  emsg = NULL;
+  GNUNET_BIO_read_close (rh, &emsg);
+  if (emsg != NULL)
+    GNUNET_free (emsg);
+  GNUNET_free (filename);
+}
+
+
+/**
+ * Method to save persistent hostlist file during hostlist client shutdown
+ * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
+ */
+static void save_hostlist_file ( int shutdown )
+{
+  char *filename;
+  struct Hostlist *pos;
+  struct GNUNET_BIO_WriteHandle * wh;
+  int ok;
+  uint32_t counter;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                             "HOSTLIST",
+                                             "HOSTLISTFILE",
+                                             &filename))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("No `%s' specified in `%s' configuration, cannot save hostlists to file.\n"),
+                  "HOSTLISTFILE", "HOSTLIST");
+      return;
+    }
+  if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
+    {
+      GNUNET_free (filename);
+      return;
+    }
+  wh = GNUNET_BIO_write_open (filename);
+  if ( NULL == wh)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("Could not open file `%s' for writing to save hostlists: %s\n"),
+                  filename,
+                 STRERROR (errno));
+      GNUNET_free (filename);
+      return;
+    }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Writing %u hostlist URIs to `%s'\n" ),
+              linked_list_size, filename);
+  /* add code to write hostlists to file using bio */
+  ok = GNUNET_YES;
+  counter = 0;
+  while (NULL != (pos = linked_list_head))
+    {
+      if ( GNUNET_YES == shutdown)
+      {
+        GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
+        linked_list_size--;
+      }
+      if (GNUNET_YES == ok)
+       {
+         if ( (GNUNET_OK !=
+               GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
+              (GNUNET_OK !=
+               GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
+              (GNUNET_OK !=
+               GNUNET_BIO_write_int64 (wh, pos->quality)) ||
+               (GNUNET_OK !=
+                GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value)) ||
+               (GNUNET_OK !=
+                GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value)) ||
+              (GNUNET_OK !=
+               GNUNET_BIO_write_int32 (wh, pos->hello_count)))
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         _("Error writing hostlist URIs to file `%s'\n"),
+                         filename);
+             ok = GNUNET_NO;
+           }
+       }
+
+      if ( GNUNET_YES == shutdown)
+        GNUNET_free (pos);
+      counter ++;
+      if ( counter >= MAX_NUMBER_HOSTLISTS) break;
+    }  
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop("# hostlist URIs written to file"),
+                         counter,
+                         GNUNET_YES);
+
+  if ( GNUNET_OK != GNUNET_BIO_write_close ( wh ) )
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+               _("Error writing hostlist URIs to file `%s'\n"),
+               filename);
+  GNUNET_free (filename);
+}
 
 /**
  * Start downloading hostlists from hostlist servers as necessary.
  */
 int
 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
 
 /**
  * Start downloading hostlists from hostlist servers as necessary.
  */
 int
 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
-                             struct GNUNET_SCHEDULER_Handle *s,
                              struct GNUNET_STATISTICS_Handle *st,
                              GNUNET_CORE_ConnectEventHandler *ch,
                              struct GNUNET_STATISTICS_Handle *st,
                              GNUNET_CORE_ConnectEventHandler *ch,
-                             GNUNET_CORE_DisconnectEventHandler *dh)
+                             GNUNET_CORE_DisconnectEventHandler *dh,
+                             GNUNET_CORE_MessageCallback *msgh,
+                             int learn)
 {
 {
+  char *filename;
+  int result;
+
   if (0 != curl_global_init (CURL_GLOBAL_WIN32))
     {
       GNUNET_break (0);
       return GNUNET_SYSERR;
     }
   if (0 != curl_global_init (CURL_GLOBAL_WIN32))
     {
       GNUNET_break (0);
       return GNUNET_SYSERR;
     }
-  transport = GNUNET_TRANSPORT_connect (s, c, NULL, NULL, NULL, NULL);
+  transport = GNUNET_TRANSPORT_connect (c, NULL, NULL, NULL, NULL, NULL);
   if (NULL == transport)
     {
       curl_global_cleanup ();
       return GNUNET_SYSERR;
     }
   cfg = c;
   if (NULL == transport)
     {
       curl_global_cleanup ();
       return GNUNET_SYSERR;
     }
   cfg = c;
-  sched = s;
   stats = st;
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
   stats = st;
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
@@ -778,11 +1517,54 @@ GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
                                             "HTTP-PROXY", 
                                             &proxy))
     proxy = NULL;
                                             "HTTP-PROXY", 
                                             &proxy))
     proxy = NULL;
-  *ch = &connect_handler;
-  *dh = &disconnect_handler;
+  stat_learning = learn;
+  *ch = &handler_connect;
+  *dh = &handler_disconnect;
+  linked_list_head = NULL;
+  linked_list_tail = NULL;
+  stat_use_bootstrap = GNUNET_YES;
+  stat_testing_hostlist = GNUNET_NO;
+  stat_testing_allowed = GNUNET_YES;
+
+  if ( GNUNET_YES == stat_learning )
+  {
+    *msgh = &handler_advertisement;
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Learning is enabled on this peer\n"));
+    load_hostlist_file ();
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Hostlists will be saved to file again in  %llums\n"),
+              (unsigned long long) SAVING_INTERVALL.rel_value);
+    ti_saving_task = GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL,
+                                               &task_hostlist_saving,
+                                               NULL);
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Learning is not enabled on this peer\n"));
+    *msgh = NULL;
+    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                                            "HOSTLIST",
+                                                            "HOSTLISTFILE",
+                                                            &filename))
+    {
+    if ( GNUNET_YES == GNUNET_DISK_file_test (filename) )
+      {
+        result = remove (filename);
+        if (result == 0)
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),filename);
+        else
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("Hostlist file `%s' could not be removed\n"),filename);
+      }
+    }
+    GNUNET_free ( filename );
+  }
   GNUNET_STATISTICS_get (stats,
                         "hostlist",
   GNUNET_STATISTICS_get (stats,
                         "hostlist",
-                        gettext_noop("Minimum time between hostlist downloads"),
+                        gettext_noop("# milliseconds between hostlist downloads"),
                         GNUNET_TIME_UNIT_MINUTES,
                         &primary_task,
                         &process_stat,
                         GNUNET_TIME_UNIT_MINUTES,
                         &primary_task,
                         &process_stat,
@@ -797,14 +1579,35 @@ GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
 void
 GNUNET_HOSTLIST_client_stop ()
 {
 void
 GNUNET_HOSTLIST_client_stop ()
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hostlist client shutdown\n");
 #if DEBUG_HOSTLIST_CLIENT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client shutdown\n");
 #endif
 #if DEBUG_HOSTLIST_CLIENT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client shutdown\n");
 #endif
-  if (current_task != GNUNET_SCHEDULER_NO_TASK)
+  if ( GNUNET_YES == stat_learning )
+    save_hostlist_file ( GNUNET_YES );
+
+  if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (ti_saving_task);
+    }
+
+  if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
+    }
+  if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
+    }
+  if (ti_download != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (ti_download);
+    }
+  if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
     {
     {
-      GNUNET_SCHEDULER_cancel (sched,
-                              current_task);
+      GNUNET_SCHEDULER_cancel (ti_check_download);
       curl_global_cleanup ();
     }
   if (transport != NULL)
       curl_global_cleanup ();
     }
   if (transport != NULL)
@@ -816,7 +1619,6 @@ GNUNET_HOSTLIST_client_stop ()
   GNUNET_free_non_null (proxy);
   proxy = NULL;
   cfg = NULL;
   GNUNET_free_non_null (proxy);
   proxy = NULL;
   cfg = NULL;
-  sched = NULL;
 }
 
 /* end of hostlist-client.c */
 }
 
 /* end of hostlist-client.c */