continue to fix extract result
[oweals/gnunet.git] / src / dht / gnunet-service-dht_neighbours.c
index 6d3ec6d8c472038194abb7c3fefb9fce78699c28..8ab39c9aaf71d132c8ec3c459197e416b82004e0 100644 (file)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2009-2013 Christian Grothoff (and other contributing authors)
+     Copyright (C) 2009-2015 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -14,8 +14,8 @@
 
      You should have received a copy of the GNU General Public License
      along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-     Boston, MA 02111-1307, USA.
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
 */
 
 /**
  * @author Christian Grothoff
  * @author Nathan Evans
  */
-
 #include "platform.h"
-#include "gnunet_block_lib.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_block_lib.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_constants.h"
 #include "gnunet_protocols.h"
@@ -46,7 +45,6 @@
 #include "gnunet-service-dht_neighbours.h"
 #include "gnunet-service-dht_nse.h"
 #include "gnunet-service-dht_routing.h"
-#include <fenv.h>
 #include "dht.h"
 
 #define LOG_TRAFFIC(kind,...) GNUNET_log_from (kind, "dht-traffic",__VA_ARGS__)
  */
 #define MAXIMUM_PENDING_PER_PEER 64
 
-/**
- * How often to update our preference levels for peers in our routing tables.
- */
-#define DHT_DEFAULT_PREFERENCE_INTERVAL GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 2)
-
 /**
  * How long at least to wait before sending another find peer request.
  */
@@ -313,7 +306,7 @@ struct PeerInfo
   struct PeerInfo *prev;
 
   /**
-   * Count of outstanding messages for peer. 
+   * Count of outstanding messages for peer.
    */
   unsigned int pending_count;
 
@@ -332,11 +325,6 @@ struct PeerInfo
    */
   struct GNUNET_CORE_TransmitHandle *th;
 
-  /**
-   * Task for scheduling preference updates
-   */
-  GNUNET_SCHEDULER_TaskIdentifier preference_task;
-
   /**
    * What is the identity of the peer?
    */
@@ -378,6 +366,35 @@ struct PeerBucket
   unsigned int peers_size;
 };
 
+
+/**
+ * Information about a peer that we would like to connect to.
+ */
+struct ConnectInfo
+{
+
+  /**
+   * Handle to active HELLO offer operation, or NULL.
+   */
+  struct GNUNET_TRANSPORT_OfferHelloHandle *oh;
+
+  /**
+   * Handle to active connectivity suggestion operation, or NULL.
+   */
+  struct GNUNET_ATS_ConnectivitySuggestHandle *sh;
+
+  /**
+   * How much would we like to connect to this peer?
+   */
+  uint32_t strength;
+};
+
+
+/**
+ * Do we cache all results that we are routing in the local datacache?
+ */
+static int cache_results;
+
 /**
  * Should routing details be logged to stderr (for debugging)?
  */
@@ -400,14 +417,21 @@ static unsigned int newly_found_peers;
 static int disable_try_connect;
 
 /**
- * The buckets.  Array of size MAX_BUCKET_SIZE.  Offset 0 means 0 bits matching.
+ * The buckets.  Array of size #MAX_BUCKETS.  Offset 0 means 0 bits matching.
  */
 static struct PeerBucket k_buckets[MAX_BUCKETS];
 
 /**
- * Hash map of all known peers, for easy removal from k_buckets on disconnect.
+ * Hash map of all CORE-connected peers, for easy removal from
+ * #k_buckets on disconnect.  Values are of type `struct PeerInfo`.
+ */
+static struct GNUNET_CONTAINER_MultiPeerMap *all_connected_peers;
+
+/**
+ * Hash map of all peers we would like to be connected to.
+ * Values are of type `struct ConnectInfo`.
  */
-static struct GNUNET_CONTAINER_MultiHashMap *all_known_peers;
+static struct GNUNET_CONTAINER_MultiPeerMap *all_desired_peers;
 
 /**
  * Maximum size for each bucket.
@@ -417,7 +441,7 @@ static unsigned int bucket_size = DEFAULT_BUCKET_SIZE;
 /**
  * Task that sends FIND PEER requests.
  */
-static GNUNET_SCHEDULER_TaskIdentifier find_peer_task;
+static struct GNUNET_SCHEDULER_Task *find_peer_task;
 
 /**
  * Identity of this peer.
@@ -425,15 +449,19 @@ static GNUNET_SCHEDULER_TaskIdentifier find_peer_task;
 static struct GNUNET_PeerIdentity my_identity;
 
 /**
- * Handle to CORE.
+ * Hash of the identity of this peer.
  */
-static struct GNUNET_CORE_Handle *coreAPI;
+static struct GNUNET_HashCode my_identity_hash;
 
 /**
- * Handle to ATS.
+ * Handle to CORE.
  */
-static struct GNUNET_ATS_PerformanceHandle *atsAPI;
+static struct GNUNET_CORE_Handle *core_api;
 
+/**
+ * Handle to ATS connectivity.
+ */
+static struct GNUNET_ATS_ConnectivityHandle *ats_ch;
 
 
 /**
@@ -444,11 +472,11 @@ static struct GNUNET_ATS_PerformanceHandle *atsAPI;
  *         on error (same hashcode)
  */
 static int
-find_bucket (const struct GNUNET_HashCode * hc)
+find_bucket (const struct GNUNET_HashCode *hc)
 {
   unsigned int bits;
 
-  bits = GNUNET_CRYPTO_hash_matching_bits (&my_identity.hashPubKey, hc);
+  bits = GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash, hc);
   if (bits == MAX_BUCKETS)
   {
     /* How can all bits match? Got my own ID? */
@@ -460,59 +488,170 @@ find_bucket (const struct GNUNET_HashCode * hc)
 
 
 /**
- * Let GNUnet core know that we like the given peer.
+ * Function called when #GNUNET_TRANSPORT_offer_hello() is done.
+ * Clean up the "oh" field in the @a cls
  *
- * @param cls the 'struct PeerInfo' of the peer
- * @param tc scheduler context.
+ * @param cls a `struct ConnectInfo`
  */
 static void
-update_core_preference (void *cls,
-                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+offer_hello_done (void *cls)
 {
-  struct PeerInfo *peer = cls;
-  uint64_t preference;
-  unsigned int matching;
-  int bucket;
+  struct ConnectInfo *ci = cls;
 
-  peer->preference_task = GNUNET_SCHEDULER_NO_TASK;
-  if ((tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
-    return;
-  matching =
-      GNUNET_CRYPTO_hash_matching_bits (&my_identity.hashPubKey,
-                                        &peer->id.hashPubKey);
-  if (matching >= 64)
-    matching = 63;
-  bucket = find_bucket (&peer->id.hashPubKey);
-  if (bucket == GNUNET_SYSERR)
-    preference = 0;
-  else
+  ci->oh = NULL;
+}
+
+
+/**
+ * Function called for all entries in #all_desired_peers to clean up.
+ *
+ * @param cls NULL
+ * @param peer peer the entry is for
+ * @param value the value to remove
+ * @return #GNUNET_YES
+ */
+static int
+free_connect_info (void *cls,
+                   const struct GNUNET_PeerIdentity *peer,
+                   void *value)
+{
+  struct ConnectInfo *ci = value;
+
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (all_desired_peers,
+                                                       peer,
+                                                       ci));
+  if (NULL != ci->sh)
   {
-    GNUNET_assert (k_buckets[bucket].peers_size != 0);
-    preference = (1LL << matching) / k_buckets[bucket].peers_size;
+    GNUNET_ATS_connectivity_suggest_cancel (ci->sh);
+    ci->sh = NULL;
   }
-  if (preference == 0)
+  if (NULL != ci->oh)
   {
-    peer->preference_task =
-        GNUNET_SCHEDULER_add_delayed (DHT_DEFAULT_PREFERENCE_INTERVAL,
-                                      &update_core_preference, peer);
+    GNUNET_TRANSPORT_offer_hello_cancel (ci->oh);
+    ci->oh = NULL;
+  }
+  GNUNET_free (ci);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Consider if we want to connect to a given peer, and if so
+ * let ATS know.  If applicable, the HELLO is offered to the
+ * TRANSPORT service.
+ *
+ * @param pid peer to consider connectivity requirements for
+ * @param h a HELLO message, or NULL
+ */
+static void
+try_connect (const struct GNUNET_PeerIdentity *pid,
+             const struct GNUNET_MessageHeader *h)
+{
+  int bucket;
+  struct GNUNET_HashCode pid_hash;
+  struct ConnectInfo *ci;
+  uint32_t strength;
+
+  GNUNET_CRYPTO_hash (pid,
+                      sizeof (struct GNUNET_PeerIdentity),
+                      &pid_hash);
+  bucket = find_bucket (&pid_hash);
+  if (bucket < 0)
+    return; /* self? */
+  ci = GNUNET_CONTAINER_multipeermap_get (all_desired_peers,
+                                          pid);
+
+  if (k_buckets[bucket].peers_size < bucket_size)
+    strength = (bucket_size - k_buckets[bucket].peers_size) * bucket;
+  else
+    strength = bucket; /* minimum value of connectivity */
+  if (GNUNET_YES ==
+      GNUNET_CONTAINER_multipeermap_contains (all_connected_peers,
+                                              pid))
+    strength *= 2; /* double for connected peers */
+  else if (k_buckets[bucket].peers_size > bucket_size)
+    strength = 0; /* bucket full, we really do not care about more */
+
+  if ( (0 == strength) &&
+       (NULL != ci) )
+  {
+    /* release request */
+    GNUNET_assert (GNUNET_YES ==
+                   free_connect_info (NULL,
+                                      pid,
+                                      ci));
     return;
   }
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Preference updates given to core"),
-                            1, GNUNET_NO);
-  GNUNET_ATS_performance_change_preference (atsAPI, &peer->id,
-                                GNUNET_ATS_PREFERENCE_BANDWIDTH,
-                                (double) preference, GNUNET_ATS_PREFERENCE_END);
-  peer->preference_task =
-      GNUNET_SCHEDULER_add_delayed (DHT_DEFAULT_PREFERENCE_INTERVAL,
-                                    &update_core_preference, peer);
+  if (NULL == ci)
+  {
+    ci = GNUNET_new (struct ConnectInfo);
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_CONTAINER_multipeermap_put (all_desired_peers,
+                                                      pid,
+                                                      ci,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  }
+  if ( (NULL != GDS_transport_handle) &&
+       (NULL != ci->oh) &&
+       (NULL != h) )
+    GNUNET_TRANSPORT_offer_hello_cancel (ci->oh);
+  if ( (NULL != GDS_transport_handle) &&
+       (NULL != h) )
+    ci->oh = GNUNET_TRANSPORT_offer_hello (GDS_transport_handle,
+                                           h,
+                                           &offer_hello_done,
+                                           ci);
+  if ( (NULL != ci->sh) &&
+       (ci->strength != strength) )
+    GNUNET_ATS_connectivity_suggest_cancel (ci->sh);
+  if (ci->strength != strength)
+    ci->sh = GNUNET_ATS_connectivity_suggest (ats_ch,
+                                              pid,
+                                              strength);
+  ci->strength = strength;
+}
 
 
+/**
+ * Function called for each peer in #all_desired_peers during
+ * #update_connect_preferences() if we have reason to adjust
+ * the strength of our desire to keep connections to certain
+ * peers.  Calls #try_connect() to update the calculations for
+ * the given @a pid.
+ *
+ * @param cls NULL
+ * @param pid peer to update
+ * @param value unused
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+update_desire_strength (void *cls,
+                        const struct GNUNET_PeerIdentity *pid,
+                        void *value)
+{
+  try_connect (pid, NULL);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Update our preferences for connectivity as given to ATS.
+ *
+ * @param cls the `struct PeerInfo` of the peer
+ * @param tc scheduler context.
+ */
+static void
+update_connect_preferences ()
+{
+  GNUNET_CONTAINER_multipeermap_iterate (all_desired_peers,
+                                         &update_desire_strength,
+                                         NULL);
 }
 
 
 /**
- * Closure for 'add_known_to_bloom'.
+ * Closure for #add_known_to_bloom().
  */
 struct BloomConstructorContext
 {
@@ -535,18 +674,26 @@ struct BloomConstructorContext
  * @param cls the 'struct BloomConstructorContext'.
  * @param key peer identity to add to the bloom filter
  * @param value value the peer information (unused)
- * @return GNUNET_YES (we should continue to iterate)
+ * @return #GNUNET_YES (we should continue to iterate)
  */
 static int
-add_known_to_bloom (void *cls, const struct GNUNET_HashCode * key, void *value)
+add_known_to_bloom (void *cls,
+                   const struct GNUNET_PeerIdentity *key,
+                   void *value)
 {
   struct BloomConstructorContext *ctx = cls;
+  struct GNUNET_HashCode key_hash;
   struct GNUNET_HashCode mh;
 
-  GNUNET_BLOCK_mingle_hash (key, ctx->bf_mutator, &mh);
+  GNUNET_CRYPTO_hash (key,
+                      sizeof (struct GNUNET_PeerIdentity),
+                      &key_hash);
+  GNUNET_BLOCK_mingle_hash (&key_hash,
+                            ctx->bf_mutator,
+                            &mh);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Adding known peer (%s) to bloomfilter for FIND PEER with mutation %u\n",
-              GNUNET_h2s (key), ctx->bf_mutator);
+              GNUNET_i2s (key), ctx->bf_mutator);
   GNUNET_CONTAINER_bloomfilter_add (ctx->bloom, &mh);
   return GNUNET_YES;
 }
@@ -558,19 +705,15 @@ add_known_to_bloom (void *cls, const struct GNUNET_HashCode * key, void *value)
  * and attempt to connect to them.
  *
  * @param cls closure for this task
- * @param tc the context under which the task is running
  */
 static void
-send_find_peer_message (void *cls,
-                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+send_find_peer_message (void *cls)
 {
   struct GNUNET_TIME_Relative next_send_time;
   struct BloomConstructorContext bcc;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
 
-  find_peer_task = GNUNET_SCHEDULER_NO_TASK;
-  if ((tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
-    return;
+  find_peer_task = NULL;
   if (newly_found_peers > bucket_size)
   {
     /* If we are finding many peers already, no need to send out our request right now! */
@@ -585,7 +728,8 @@ send_find_peer_message (void *cls,
   bcc.bloom =
       GNUNET_CONTAINER_bloomfilter_init (NULL, DHT_BLOOM_SIZE,
                                          GNUNET_CONSTANTS_BLOOMFILTER_K);
-  GNUNET_CONTAINER_multihashmap_iterate (all_known_peers, &add_known_to_bloom,
+  GNUNET_CONTAINER_multipeermap_iterate (all_connected_peers,
+                                         &add_known_to_bloom,
                                          &bcc);
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# FIND PEER messages initiated"), 1,
@@ -597,7 +741,7 @@ send_find_peer_message (void *cls,
   GDS_NEIGHBOURS_handle_get (GNUNET_BLOCK_TYPE_DHT_HELLO,
                              GNUNET_DHT_RO_FIND_PEER,
                              FIND_PEER_REPLICATION_LEVEL, 0,
-                             &my_identity.hashPubKey, NULL, 0, bcc.bloom,
+                             &my_identity_hash, NULL, 0, bcc.bloom,
                              bcc.bf_mutator, peer_bf);
   GNUNET_CONTAINER_bloomfilter_free (peer_bf);
   GNUNET_CONTAINER_bloomfilter_free (bcc.bloom);
@@ -621,52 +765,64 @@ send_find_peer_message (void *cls,
  * @param peer peer identity this notification is about
  */
 static void
-handle_core_connect (void *cls, const struct GNUNET_PeerIdentity *peer)
+handle_core_connect (void *cls,
+                     const struct GNUNET_PeerIdentity *peer)
 {
   struct PeerInfo *ret;
+  struct GNUNET_HashCode phash;
   int peer_bucket;
 
   /* Check for connect to self message */
   if (0 == memcmp (&my_identity, peer, sizeof (struct GNUNET_PeerIdentity)))
     return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connected %s to %s\n",
-              GNUNET_i2s (&my_identity), GNUNET_h2s (&peer->hashPubKey));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Connected to %s\n",
+              GNUNET_i2s (peer));
   if (GNUNET_YES ==
-      GNUNET_CONTAINER_multihashmap_contains (all_known_peers,
-                                              &peer->hashPubKey))
+      GNUNET_CONTAINER_multipeermap_contains (all_connected_peers,
+                                              peer))
   {
     GNUNET_break (0);
     return;
   }
-  GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# peers connected"), 1,
+  GNUNET_STATISTICS_update (GDS_stats,
+                            gettext_noop ("# peers connected"),
+                            1,
                             GNUNET_NO);
-  peer_bucket = find_bucket (&peer->hashPubKey);
+  GNUNET_CRYPTO_hash (peer,
+                     sizeof (struct GNUNET_PeerIdentity),
+                     &phash);
+  peer_bucket = find_bucket (&phash);
   GNUNET_assert ((peer_bucket >= 0) && (peer_bucket < MAX_BUCKETS));
-  ret = GNUNET_malloc (sizeof (struct PeerInfo));
+  ret = GNUNET_new (struct PeerInfo);
 #if 0
   ret->latency = latency;
   ret->distance = distance;
 #endif
   ret->id = *peer;
   GNUNET_CONTAINER_DLL_insert_tail (k_buckets[peer_bucket].head,
-                                    k_buckets[peer_bucket].tail, ret);
+                                    k_buckets[peer_bucket].tail,
+                                    ret);
   k_buckets[peer_bucket].peers_size++;
-  closest_bucket = GNUNET_MAX (closest_bucket, peer_bucket);
-  if ((peer_bucket > 0) && (k_buckets[peer_bucket].peers_size <= bucket_size))
+  closest_bucket = GNUNET_MAX (closest_bucket,
+                               peer_bucket);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multipeermap_put (all_connected_peers,
+                                                    peer,
+                                                    ret,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  if ( (peer_bucket > 0) &&
+       (k_buckets[peer_bucket].peers_size <= bucket_size))
   {
-    ret->preference_task =
-        GNUNET_SCHEDULER_add_now (&update_core_preference, ret);
+    update_connect_preferences ();
     newly_found_peers++;
   }
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (all_known_peers,
-                                                    &peer->hashPubKey, ret,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  if (1 == GNUNET_CONTAINER_multihashmap_size (all_known_peers) &&
+  if (1 == GNUNET_CONTAINER_multipeermap_size (all_connected_peers) &&
       (GNUNET_YES != disable_try_connect))
   {
     /* got a first connection, good time to start with FIND PEER requests... */
-    find_peer_task = GNUNET_SCHEDULER_add_now (&send_find_peer_message, NULL);
+    find_peer_task = GNUNET_SCHEDULER_add_now (&send_find_peer_message,
+                                               NULL);
   }
 }
 
@@ -678,46 +834,53 @@ handle_core_connect (void *cls, const struct GNUNET_PeerIdentity *peer)
  * @param peer peer identity this notification is about
  */
 static void
-handle_core_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
+handle_core_disconnect (void *cls,
+                       const struct GNUNET_PeerIdentity *peer)
 {
   struct PeerInfo *to_remove;
   int current_bucket;
   struct P2PPendingMessage *pos;
   unsigned int discarded;
+  struct GNUNET_HashCode phash;
 
   /* Check for disconnect from self message */
-  if (0 == memcmp (&my_identity, peer, sizeof (struct GNUNET_PeerIdentity)))
+  if (0 == memcmp (&my_identity,
+                   peer,
+                   sizeof (struct GNUNET_PeerIdentity)))
     return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Disconnected %s from %s\n",
-              GNUNET_i2s (&my_identity), GNUNET_h2s (&peer->hashPubKey));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Disconnected %s\n",
+              GNUNET_i2s (peer));
   to_remove =
-      GNUNET_CONTAINER_multihashmap_get (all_known_peers, &peer->hashPubKey);
+      GNUNET_CONTAINER_multipeermap_get (all_connected_peers,
+                                         peer);
   if (NULL == to_remove)
   {
     GNUNET_break (0);
     return;
   }
-  GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# peers connected"), -1,
+  GNUNET_STATISTICS_update (GDS_stats,
+                            gettext_noop ("# peers connected"),
+                            -1,
                             GNUNET_NO);
   GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (all_known_peers,
-                                                       &peer->hashPubKey,
+                 GNUNET_CONTAINER_multipeermap_remove (all_connected_peers,
+                                                       peer,
                                                        to_remove));
-  if (GNUNET_SCHEDULER_NO_TASK != to_remove->preference_task)
-  {
-    GNUNET_SCHEDULER_cancel (to_remove->preference_task);
-    to_remove->preference_task = GNUNET_SCHEDULER_NO_TASK;
-  }
-  current_bucket = find_bucket (&to_remove->id.hashPubKey);
+  GNUNET_CRYPTO_hash (peer,
+                     sizeof (struct GNUNET_PeerIdentity),
+                     &phash);
+  current_bucket = find_bucket (&phash);
   GNUNET_assert (current_bucket >= 0);
   GNUNET_CONTAINER_DLL_remove (k_buckets[current_bucket].head,
-                               k_buckets[current_bucket].tail, to_remove);
+                               k_buckets[current_bucket].tail,
+                               to_remove);
   GNUNET_assert (k_buckets[current_bucket].peers_size > 0);
   k_buckets[current_bucket].peers_size--;
-  while ((closest_bucket > 0) && (k_buckets[closest_bucket].peers_size == 0))
+  while ( (closest_bucket > 0) &&
+          (0 == k_buckets[closest_bucket].peers_size) )
     closest_bucket--;
-
-  if (to_remove->th != NULL)
+  if (NULL != to_remove->th)
   {
     GNUNET_CORE_notify_transmit_ready_cancel (to_remove->th);
     to_remove->th = NULL;
@@ -725,14 +888,18 @@ handle_core_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
   discarded = 0;
   while (NULL != (pos = to_remove->head))
   {
-    GNUNET_CONTAINER_DLL_remove (to_remove->head, to_remove->tail, pos);
+    GNUNET_CONTAINER_DLL_remove (to_remove->head,
+                                 to_remove->tail,
+                                 pos);
     discarded++;
     GNUNET_free (pos);
   }
+  if (k_buckets[current_bucket].peers_size < bucket_size)
+    update_connect_preferences ();
   GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop
-                            ("# Queued messages discarded (peer disconnected)"),
-                            discarded, GNUNET_NO);
+                            gettext_noop ("# Queued messages discarded (peer disconnected)"),
+                            discarded,
+                            GNUNET_NO);
   GNUNET_free (to_remove);
 }
 
@@ -742,12 +909,14 @@ handle_core_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
  * out to the destination.
  *
  * @param cls the 'struct PeerInfo' of the target peer
- * @param size number of bytes available in buf
+ * @param size number of bytes available in @a buf
  * @param buf where the callee should write the message
- * @return number of bytes written to buf
+ * @return number of bytes written to @a buf
  */
 static size_t
-core_transmit_notify (void *cls, size_t size, void *buf)
+core_transmit_notify (void *cls,
+                      size_t size,
+                      void *buf)
 {
   struct PeerInfo *peer = cls;
   char *cbuf = buf;
@@ -759,20 +928,25 @@ core_transmit_notify (void *cls, size_t size, void *buf)
   while ((NULL != (pending = peer->head)) &&
          (0 == GNUNET_TIME_absolute_get_remaining (pending->timeout).rel_value_us))
   {
+    GNUNET_STATISTICS_update (GDS_stats,
+                              gettext_noop
+                              ("# Messages dropped (CORE timeout)"),
+                              1,
+                              GNUNET_NO);
     peer->pending_count--;
     GNUNET_CONTAINER_DLL_remove (peer->head, peer->tail, pending);
     GNUNET_free (pending);
   }
-  if (pending == NULL)
+  if (NULL == pending)
   {
     /* no messages pending */
     return 0;
   }
-  if (buf == NULL)
+  if (NULL == buf)
   {
     peer->th =
-        GNUNET_CORE_notify_transmit_ready (coreAPI, GNUNET_NO,
-                                           pending->importance,
+        GNUNET_CORE_notify_transmit_ready (core_api, GNUNET_NO,
+                                           GNUNET_CORE_PRIO_BEST_EFFORT,
                                            GNUNET_TIME_absolute_get_remaining
                                            (pending->timeout), &peer->id,
                                            ntohs (pending->msg->size),
@@ -791,17 +965,25 @@ core_transmit_notify (void *cls, size_t size, void *buf)
     memcpy (&cbuf[off], pending->msg, msize);
     off += msize;
     peer->pending_count--;
-    GNUNET_CONTAINER_DLL_remove (peer->head, peer->tail, pending);
+    GNUNET_CONTAINER_DLL_remove (peer->head,
+                                peer->tail,
+                                pending);
     GNUNET_free (pending);
   }
-  if (peer->head != NULL)
+  if (NULL != (pending = peer->head))
   {
+    /* technically redundant, but easier to read and
+       avoids bogus gcc warning... */
+    msize = ntohs (pending->msg->size);
     peer->th =
-        GNUNET_CORE_notify_transmit_ready (coreAPI, GNUNET_NO,
-                                           pending->importance,
-                                           GNUNET_TIME_absolute_get_remaining
-                                           (pending->timeout), &peer->id, msize,
-                                           &core_transmit_notify, peer);
+      GNUNET_CORE_notify_transmit_ready (core_api,
+                                        GNUNET_NO,
+                                        GNUNET_CORE_PRIO_BEST_EFFORT,
+                                        GNUNET_TIME_absolute_get_remaining (pending->timeout),
+                                        &peer->id,
+                                        msize,
+                                        &core_transmit_notify,
+                                        peer);
     GNUNET_break (NULL != peer->th);
   }
   return off;
@@ -827,12 +1009,14 @@ process_peer_queue (struct PeerInfo *peer)
                             ("# Bytes of bandwidth requested from core"),
                             ntohs (pending->msg->size), GNUNET_NO);
   peer->th =
-      GNUNET_CORE_notify_transmit_ready (coreAPI, GNUNET_NO,
-                                         pending->importance,
+      GNUNET_CORE_notify_transmit_ready (core_api, GNUNET_NO,
+                                         GNUNET_CORE_PRIO_BEST_EFFORT,
                                          GNUNET_TIME_absolute_get_remaining
-                                         (pending->timeout), &peer->id,
+                                         (pending->timeout),
+                                         &peer->id,
                                          ntohs (pending->msg->size),
-                                         &core_transmit_notify, peer);
+                                         &core_transmit_notify,
+                                         peer);
   GNUNET_break (NULL != peer->th);
 }
 
@@ -889,12 +1073,15 @@ get_forward_count (uint32_t hop_count, uint32_t target_replication)
  * Differences in the lower bits must count stronger than differences
  * in the higher bits.
  *
+ * @param target
+ * @param have
  * @return 0 if have==target, otherwise a number
  *           that is larger as the distance between
  *           the two hash codes increases
  */
 static unsigned int
-get_distance (const struct GNUNET_HashCode * target, const struct GNUNET_HashCode * have)
+get_distance (const struct GNUNET_HashCode *target,
+             const struct GNUNET_HashCode *have)
 {
   unsigned int bucket;
   unsigned int msb;
@@ -950,11 +1137,11 @@ get_distance (const struct GNUNET_HashCode * target, const struct GNUNET_HashCod
  *
  * @param key hash code to check closeness to
  * @param bloom bloomfilter, exclude these entries from the decision
- * @return GNUNET_YES if node location is closest,
- *         GNUNET_NO otherwise.
+ * @return #GNUNET_YES if node location is closest,
+ *         #GNUNET_NO otherwise.
  */
 static int
-am_closest_peer (const struct GNUNET_HashCode * key,
+am_closest_peer (const struct GNUNET_HashCode *key,
                  const struct GNUNET_CONTAINER_BloomFilter *bloom)
 {
   int bits;
@@ -962,24 +1149,28 @@ am_closest_peer (const struct GNUNET_HashCode * key,
   int bucket_num;
   int count;
   struct PeerInfo *pos;
+  struct GNUNET_HashCode phash;
 
-  if (0 == memcmp (&my_identity.hashPubKey, key, sizeof (struct GNUNET_HashCode)))
+  if (0 == memcmp (&my_identity_hash, key, sizeof (struct GNUNET_HashCode)))
     return GNUNET_YES;
   bucket_num = find_bucket (key);
   GNUNET_assert (bucket_num >= 0);
-  bits = GNUNET_CRYPTO_hash_matching_bits (&my_identity.hashPubKey, key);
+  bits = GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash, key);
   pos = k_buckets[bucket_num].head;
   count = 0;
-  while ((pos != NULL) && (count < bucket_size))
+  while ((NULL != pos) && (count < bucket_size))
   {
-    if ((bloom != NULL) &&
+    GNUNET_CRYPTO_hash (&pos->id,
+                       sizeof (struct GNUNET_PeerIdentity),
+                       &phash);
+    if ((NULL != bloom) &&
         (GNUNET_YES ==
-         GNUNET_CONTAINER_bloomfilter_test (bloom, &pos->id.hashPubKey)))
+         GNUNET_CONTAINER_bloomfilter_test (bloom, &phash)))
     {
       pos = pos->next;
       continue;                 /* Skip already checked entries */
     }
-    other_bits = GNUNET_CRYPTO_hash_matching_bits (&pos->id.hashPubKey, key);
+    other_bits = GNUNET_CRYPTO_hash_matching_bits (&phash, key);
     if (other_bits > bits)
       return GNUNET_NO;
     if (other_bits == bits)     /* We match the same number of bits */
@@ -1009,8 +1200,9 @@ am_closest_peer (const struct GNUNET_HashCode * key,
  * @return Peer to route to, or NULL on error
  */
 static struct PeerInfo *
-select_peer (const struct GNUNET_HashCode * key,
-             const struct GNUNET_CONTAINER_BloomFilter *bloom, uint32_t hops)
+select_peer (const struct GNUNET_HashCode *key,
+             const struct GNUNET_CONTAINER_BloomFilter *bloom,
+             uint32_t hops)
 {
   unsigned int bc;
   unsigned int count;
@@ -1019,6 +1211,7 @@ select_peer (const struct GNUNET_HashCode * key,
   unsigned int dist;
   unsigned int smallest_distance;
   struct PeerInfo *chosen;
+  struct GNUNET_HashCode phash;
 
   if (hops >= GDS_NSE_get ())
   {
@@ -1031,11 +1224,14 @@ select_peer (const struct GNUNET_HashCode * key,
       count = 0;
       while ((pos != NULL) && (count < bucket_size))
       {
+       GNUNET_CRYPTO_hash (&pos->id,
+                           sizeof (struct GNUNET_PeerIdentity),
+                           &phash);
         if ((bloom == NULL) ||
             (GNUNET_NO ==
-             GNUNET_CONTAINER_bloomfilter_test (bloom, &pos->id.hashPubKey)))
+             GNUNET_CONTAINER_bloomfilter_test (bloom, &phash)))
         {
-          dist = get_distance (key, &pos->id.hashPubKey);
+          dist = get_distance (key, &phash);
           if (dist < smallest_distance)
           {
             chosen = pos;
@@ -1051,7 +1247,7 @@ select_peer (const struct GNUNET_HashCode * key,
                                     gettext_noop
                                     ("# Peers excluded from routing due to Bloomfilter"),
                                     1, GNUNET_NO);
-          dist = get_distance (key, &pos->id.hashPubKey);
+          dist = get_distance (key, &phash);
           if (dist < smallest_distance)
           {
             chosen = NULL;
@@ -1077,9 +1273,12 @@ select_peer (const struct GNUNET_HashCode * key,
     pos = k_buckets[bc].head;
     while ((pos != NULL) && (count < bucket_size))
     {
+      GNUNET_CRYPTO_hash (&pos->id,
+                         sizeof (struct GNUNET_PeerIdentity),
+                         &phash);
       if ((bloom != NULL) &&
           (GNUNET_YES ==
-           GNUNET_CONTAINER_bloomfilter_test (bloom, &pos->id.hashPubKey)))
+           GNUNET_CONTAINER_bloomfilter_test (bloom, &phash)))
       {
         GNUNET_STATISTICS_update (GDS_stats,
                                   gettext_noop
@@ -1109,9 +1308,12 @@ select_peer (const struct GNUNET_HashCode * key,
   {
     for (pos = k_buckets[bc].head; ((pos != NULL) && (count < bucket_size)); pos = pos->next)
     {
+      GNUNET_CRYPTO_hash (&pos->id,
+                         sizeof (struct GNUNET_PeerIdentity),
+                         &phash);
       if ((bloom != NULL) &&
           (GNUNET_YES ==
-           GNUNET_CONTAINER_bloomfilter_test (bloom, &pos->id.hashPubKey)))
+           GNUNET_CONTAINER_bloomfilter_test (bloom, &phash)))
       {
         continue;               /* Ignore bloomfiltered peers */
       }
@@ -1147,9 +1349,10 @@ get_target_peers (const struct GNUNET_HashCode *key,
   unsigned int off;
   struct PeerInfo **rtargets;
   struct PeerInfo *nxt;
+  struct GNUNET_HashCode nhash;
 
   GNUNET_assert (NULL != bloom);
-  ret = get_forward_count (hop_count, target_replication);  
+  ret = get_forward_count (hop_count, target_replication);
   if (0 == ret)
   {
     *targets = NULL;
@@ -1162,15 +1365,21 @@ get_target_peers (const struct GNUNET_HashCode *key,
     if (NULL == nxt)
       break;
     rtargets[off] = nxt;
+    GNUNET_CRYPTO_hash (&nxt->id,
+                       sizeof (struct GNUNET_PeerIdentity),
+                       &nhash);
     GNUNET_break (GNUNET_NO ==
                   GNUNET_CONTAINER_bloomfilter_test (bloom,
-                                                     &nxt->id.hashPubKey));
-    GNUNET_CONTAINER_bloomfilter_add (bloom, &rtargets[off]->id.hashPubKey);
+                                                     &nhash));
+    GNUNET_CONTAINER_bloomfilter_add (bloom, &nhash);
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Selected %u/%u peers at hop %u for %s (target was %u)\n", off,
-              GNUNET_CONTAINER_multihashmap_size (all_known_peers),
-              (unsigned int) hop_count, GNUNET_h2s (key), ret);
+              "Selected %u/%u peers at hop %u for %s (target was %u)\n",
+              off,
+              GNUNET_CONTAINER_multipeermap_size (all_connected_peers),
+              (unsigned int) hop_count,
+              GNUNET_h2s (key),
+              ret);
   if (0 == off)
   {
     GNUNET_free (rtargets);
@@ -1180,7 +1389,7 @@ get_target_peers (const struct GNUNET_HashCode *key,
   *targets = rtargets;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Forwarding query `%s' to %u peers (goal was %u peers)\n",
-             GNUNET_h2s (key), 
+             GNUNET_h2s (key),
              off,
              ret);
   return off;
@@ -1205,8 +1414,9 @@ get_target_peers (const struct GNUNET_HashCode *key,
  * @param put_path peers this request has traversed so far (if tracked)
  * @param data payload to store
  * @param data_size number of bytes in @a data
+ * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
  */
-void
+int
 GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
                            enum GNUNET_DHT_RouteOption options,
                            uint32_t desired_replication_level,
@@ -1226,12 +1436,14 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
   size_t msize;
   struct PeerPutMessage *ppm;
   struct GNUNET_PeerIdentity *pp;
+  struct GNUNET_HashCode thash;
+  unsigned int skip_count;
 
   GNUNET_assert (NULL != bf);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Adding myself (%s) to PUT bloomfilter for %s\n",
               GNUNET_i2s (&my_identity), GNUNET_h2s (key));
-  GNUNET_CONTAINER_bloomfilter_add (bf, &my_identity.hashPubKey);
+  GNUNET_CONTAINER_bloomfilter_add (bf, &my_identity_hash);
   GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# PUT requests routed"),
                             1, GNUNET_NO);
   target_count =
@@ -1243,34 +1455,38 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
                 "Routing PUT for %s terminates after %u hops at %s\n",
                 GNUNET_h2s (key), (unsigned int) hop_count,
                 GNUNET_i2s (&my_identity));
-    return;
+    return GNUNET_NO;
   }
   msize =
       put_path_length * sizeof (struct GNUNET_PeerIdentity) + data_size +
       sizeof (struct PeerPutMessage);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
   {
     put_path_length = 0;
     msize = data_size + sizeof (struct PeerPutMessage);
   }
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     GNUNET_free (targets);
-    return;
+    return GNUNET_NO;
   }
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop
                             ("# PUT messages queued for transmission"),
                             target_count, GNUNET_NO);
+  skip_count = 0;
   for (i = 0; i < target_count; i++)
   {
     target = targets[i];
     if (target->pending_count >= MAXIMUM_PENDING_PER_PEER)
     {
-      GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# P2P messages dropped due to full queue"),
+      /* skip */
+      GNUNET_STATISTICS_update (GDS_stats,
+                                gettext_noop ("# P2P messages dropped due to full queue"),
                                1, GNUNET_NO);
-      continue; /* skip */
+      skip_count++;
+      continue;
     }
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Routing PUT for %s after %u hops to %s\n", GNUNET_h2s (key),
@@ -1288,9 +1504,12 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
     ppm->desired_replication_level = htonl (desired_replication_level);
     ppm->put_path_length = htonl (put_path_length);
     ppm->expiration_time = GNUNET_TIME_absolute_hton (expiration_time);
+    GNUNET_CRYPTO_hash (&target->id,
+                       sizeof (struct GNUNET_PeerIdentity),
+                       &thash);
     GNUNET_break (GNUNET_YES ==
                   GNUNET_CONTAINER_bloomfilter_test (bf,
-                                                     &target->id.hashPubKey));
+                                                     &thash));
     GNUNET_assert (GNUNET_OK ==
                    GNUNET_CONTAINER_bloomfilter_get_raw_data (bf,
                                                               ppm->bloomfilter,
@@ -1305,6 +1524,7 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
     process_peer_queue (target);
   }
   GNUNET_free (targets);
+  return (skip_count < target_count) ? GNUNET_OK : GNUNET_NO;
 }
 
 
@@ -1324,8 +1544,9 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
  * @param reply_bf bloomfilter to filter duplicates
  * @param reply_bf_mutator mutator for @a reply_bf
  * @param peer_bf filter for peers not to select (again)
+ * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
  */
-void
+int
 GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                            enum GNUNET_DHT_RouteOption options,
                            uint32_t desired_replication_level,
@@ -1344,6 +1565,8 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
   struct PeerGetMessage *pgm;
   char *xq;
   size_t reply_bf_size;
+  struct GNUNET_HashCode thash;
+  unsigned int skip_count;
 
   GNUNET_assert (NULL != peer_bf);
   GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# GET requests routed"),
@@ -1354,14 +1577,14 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Adding myself (%s) to GET bloomfilter for %s\n",
               GNUNET_i2s (&my_identity), GNUNET_h2s (key));
-  GNUNET_CONTAINER_bloomfilter_add (peer_bf, &my_identity.hashPubKey);
+  GNUNET_CONTAINER_bloomfilter_add (peer_bf, &my_identity_hash);
   if (0 == target_count)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Routing GET for %s terminates after %u hops at %s\n",
                 GNUNET_h2s (key), (unsigned int) hop_count,
                 GNUNET_i2s (&my_identity));
-    return;
+    return GNUNET_NO;
   }
   reply_bf_size = GNUNET_CONTAINER_bloomfilter_get_size (reply_bf);
   msize = xquery_size + sizeof (struct PeerGetMessage) + reply_bf_size;
@@ -1369,21 +1592,25 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
   {
     GNUNET_break (0);
     GNUNET_free (targets);
-    return;
+    return GNUNET_NO;
   }
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop
                             ("# GET messages queued for transmission"),
                             target_count, GNUNET_NO);
   /* forward request */
+  skip_count = 0;
   for (i = 0; i < target_count; i++)
   {
     target = targets[i];
     if (target->pending_count >= MAXIMUM_PENDING_PER_PEER)
     {
-      GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# P2P messages dropped due to full queue"),
+      /* skip */
+      GNUNET_STATISTICS_update (GDS_stats,
+                                gettext_noop ("# P2P messages dropped due to full queue"),
                                1, GNUNET_NO);
-      continue; /* skip */
+      skip_count++;
+      continue;
     }
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Routing GET for %s after %u hops to %s\n", GNUNET_h2s (key),
@@ -1401,9 +1628,12 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
     pgm->desired_replication_level = htonl (desired_replication_level);
     pgm->xquery_size = htonl (xquery_size);
     pgm->bf_mutator = reply_bf_mutator;
+    GNUNET_CRYPTO_hash (&target->id,
+                       sizeof (struct GNUNET_PeerIdentity),
+                       &thash);
     GNUNET_break (GNUNET_YES ==
                   GNUNET_CONTAINER_bloomfilter_test (peer_bf,
-                                                     &target->id.hashPubKey));
+                                                     &thash));
     GNUNET_assert (GNUNET_OK ==
                    GNUNET_CONTAINER_bloomfilter_get_raw_data (peer_bf,
                                                               pgm->bloomfilter,
@@ -1422,6 +1652,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
     process_peer_queue (target);
   }
   GNUNET_free (targets);
+  return (skip_count < target_count) ? GNUNET_OK : GNUNET_NO;
 }
 
 
@@ -1445,12 +1676,13 @@ void
 GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
                              enum GNUNET_BLOCK_Type type,
                              struct GNUNET_TIME_Absolute expiration_time,
-                             const struct GNUNET_HashCode * key,
+                             const struct GNUNET_HashCode *key,
                              unsigned int put_path_length,
                              const struct GNUNET_PeerIdentity *put_path,
                              unsigned int get_path_length,
                              const struct GNUNET_PeerIdentity *get_path,
-                             const void *data, size_t data_size)
+                             const void *data,
+                             size_t data_size)
 {
   struct PeerInfo *pi;
   struct P2PPendingMessage *pending;
@@ -1472,7 +1704,8 @@ GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
     GNUNET_break (0);
     return;
   }
-  pi = GNUNET_CONTAINER_multihashmap_get (all_known_peers, &target->hashPubKey);
+  pi = GNUNET_CONTAINER_multipeermap_get (all_connected_peers,
+                                          target);
   if (NULL == pi)
   {
     /* peer disconnected in the meantime, drop reply */
@@ -1525,6 +1758,9 @@ core_init (void *cls,
            const struct GNUNET_PeerIdentity *identity)
 {
   my_identity = *identity;
+  GNUNET_CRYPTO_hash (identity,
+                     sizeof (struct GNUNET_PeerIdentity),
+                     &my_identity_hash);
 }
 
 
@@ -1539,8 +1775,7 @@ core_init (void *cls,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_dht_p2p_put (void *cls, 
-                   const struct GNUNET_PeerIdentity *peer,
+handle_dht_p2p_put (void *cls, const struct GNUNET_PeerIdentity *peer,
                     const struct GNUNET_MessageHeader *message)
 {
   const struct PeerPutMessage *put;
@@ -1552,6 +1787,8 @@ handle_dht_p2p_put (void *cls,
   enum GNUNET_DHT_RouteOption options;
   struct GNUNET_CONTAINER_BloomFilter *bf;
   struct GNUNET_HashCode test_key;
+  struct GNUNET_HashCode phash;
+  int forwarded;
 
   msize = ntohs (message->size);
   if (msize < sizeof (struct PeerPutMessage))
@@ -1582,19 +1819,21 @@ handle_dht_p2p_put (void *cls,
   payload_size =
       msize - (sizeof (struct PeerPutMessage) +
                putlen * sizeof (struct GNUNET_PeerIdentity));
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "PUT for `%s' from %s\n",
               GNUNET_h2s (&put->key), GNUNET_i2s (peer));
+  GNUNET_CRYPTO_hash (peer, sizeof (struct GNUNET_PeerIdentity), &phash);
   if (GNUNET_YES == log_route_details_stderr)
   {
     char *tmp;
 
     tmp = GNUNET_strdup (GNUNET_i2s (&my_identity));
     LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
-                "XDHT PUT %s: %s->%s (%u, %u=>%u)\n", 
+                 "R5N PUT %s: %s->%s (%u, %u=>%u)\n",
                  GNUNET_h2s (&put->key), GNUNET_i2s (peer), tmp,
                  ntohl(put->hop_count),
-                 GNUNET_CRYPTO_hash_matching_bits (&peer->hashPubKey, &put->key),
-                 GNUNET_CRYPTO_hash_matching_bits (&my_identity.hashPubKey, &put->key)
+                 GNUNET_CRYPTO_hash_matching_bits (&phash, &put->key),
+                 GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash, &put->key)
                 );
     GNUNET_free (tmp);
   }
@@ -1605,7 +1844,12 @@ handle_dht_p2p_put (void *cls,
   case GNUNET_YES:
     if (0 != memcmp (&test_key, &put->key, sizeof (struct GNUNET_HashCode)))
     {
+      char *put_s = GNUNET_strdup (GNUNET_h2s_full (&put->key));
       GNUNET_break_op (0);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "PUT with key `%s' for block with key %s\n",
+                  put_s, GNUNET_h2s_full (&test_key));
+      GNUNET_free (put_s);
       return GNUNET_YES;
     }
     break;
@@ -1620,6 +1864,7 @@ handle_dht_p2p_put (void *cls,
   {
     switch (GNUNET_BLOCK_evaluate (GDS_block_context,
                                    ntohl (put->type),
+                                   GNUNET_BLOCK_EO_NONE,
                                    NULL,    /* query */
                                    NULL, 0, /* bloom filer */
                                    NULL, 0, /* xquery */
@@ -1644,7 +1889,7 @@ handle_dht_p2p_put (void *cls,
   bf = GNUNET_CONTAINER_bloomfilter_init (put->bloomfilter, DHT_BLOOM_SIZE,
                                           GNUNET_CONSTANTS_BLOOMFILTER_K);
   GNUNET_break_op (GNUNET_YES ==
-                   GNUNET_CONTAINER_bloomfilter_test (bf, &peer->hashPubKey));
+                   GNUNET_CONTAINER_bloomfilter_test (bf, &phash));
   {
     struct GNUNET_PeerIdentity pp[putlen + 1];
 
@@ -1669,13 +1914,20 @@ handle_dht_p2p_put (void *cls,
                                 (put->expiration_time), &put->key, putlen, pp,
                                 ntohl (put->type), payload_size, payload);
     /* route to other peers */
-    GDS_NEIGHBOURS_handle_put (ntohl (put->type), options,
-                               ntohl (put->desired_replication_level),
-                               GNUNET_TIME_absolute_ntoh (put->expiration_time),
-                               ntohl (put->hop_count), bf, &put->key, putlen,
-                               pp, payload, payload_size);
+    forwarded = GDS_NEIGHBOURS_handle_put (ntohl (put->type), options,
+                                           ntohl (put->desired_replication_level),
+                                           GNUNET_TIME_absolute_ntoh (put->expiration_time),
+                                           ntohl (put->hop_count), bf,
+                                           &put->key,
+                                           putlen,
+                                           pp,
+                                           payload,
+                                           payload_size);
     /* notify monitoring clients */
-    GDS_CLIENTS_process_put (options,
+    GDS_CLIENTS_process_put (options
+                             | ( (GNUNET_OK == forwarded)
+                                 ? GNUNET_DHT_RO_LAST_HOP
+                                 : 0 ),
                              ntohl (put->type),
                              ntohl (put->hop_count),
                              ntohl (put->desired_replication_level),
@@ -1708,13 +1960,14 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
   struct PeerBucket *bucket;
   struct PeerInfo *peer;
   unsigned int choice;
+  struct GNUNET_HashCode phash;
   struct GNUNET_HashCode mhash;
   const struct GNUNET_HELLO_Message *hello;
 
   /* first, check about our own HELLO */
   if (NULL != GDS_my_hello)
   {
-    GNUNET_BLOCK_mingle_hash (&my_identity.hashPubKey, bf_mutator, &mhash);
+    GNUNET_BLOCK_mingle_hash (&my_identity_hash, bf_mutator, &mhash);
     if ((NULL == bf) ||
         (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (bf, &mhash)))
     {
@@ -1743,7 +1996,7 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
   }
 
   /* then, also consider sending a random HELLO from the closest bucket */
-  if (0 == memcmp (&my_identity.hashPubKey, key, sizeof (struct GNUNET_HashCode)))
+  if (0 == memcmp (&my_identity_hash, key, sizeof (struct GNUNET_HashCode)))
     bucket_idx = closest_bucket;
   else
     bucket_idx = GNUNET_MIN (closest_bucket, find_bucket (key));
@@ -1757,7 +2010,7 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
   peer = bucket->head;
   while (choice > 0)
   {
-    GNUNET_assert (peer != NULL);
+    GNUNET_assert (NULL != peer);
     peer = peer->next;
     choice--;
   }
@@ -1769,7 +2022,8 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
       return;                   /* no non-masked peer available */
     if (peer == NULL)
       peer = bucket->head;
-    GNUNET_BLOCK_mingle_hash (&peer->id.hashPubKey, bf_mutator, &mhash);
+    GNUNET_CRYPTO_hash (&peer->id, sizeof (struct GNUNET_PeerIdentity), &phash);
+    GNUNET_BLOCK_mingle_hash (&phash, bf_mutator, &mhash);
     hello = GDS_HELLO_get (&peer->id);
   }
   while ((hello == NULL) ||
@@ -1788,11 +2042,12 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
  * @param cls closure
  * @param peer sender of the request
  * @param message message
- * @return GNUNET_OK to keep the connection open,
- *         GNUNET_SYSERR to close it (signal serious error)
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
+handle_dht_p2p_get (void *cls,
+                    const struct GNUNET_PeerIdentity *peer,
                     const struct GNUNET_MessageHeader *message)
 {
   struct PeerGetMessage *get;
@@ -1805,6 +2060,8 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
   struct GNUNET_CONTAINER_BloomFilter *reply_bf;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
   const char *xquery;
+  struct GNUNET_HashCode phash;
+  int forwarded;
 
   GNUNET_break (0 !=
                 memcmp (peer, &my_identity,
@@ -1834,19 +2091,21 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# P2P GET bytes received"), msize,
                             GNUNET_NO);
+  GNUNET_CRYPTO_hash (peer,
+                      sizeof (struct GNUNET_PeerIdentity),
+                      &phash);
   if (GNUNET_YES == log_route_details_stderr)
   {
     char *tmp;
 
     tmp = GNUNET_strdup (GNUNET_i2s (&my_identity));
     LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
-                 "XDHT GET %s: %s->%s (%u, %u=>%u) xq: %.*s\n", 
+                 "R5N GET %s: %s->%s (%u, %u=>%u) xq: %.*s\n",
                  GNUNET_h2s (&get->key), GNUNET_i2s (peer), tmp,
                  ntohl(get->hop_count),
-                 GNUNET_CRYPTO_hash_matching_bits (&peer->hashPubKey, &get->key),
-                 GNUNET_CRYPTO_hash_matching_bits (&my_identity.hashPubKey, &get->key),
-                 ntohl(get->xquery_size), xquery
-                );
+                 GNUNET_CRYPTO_hash_matching_bits (&phash, &get->key),
+                 GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash, &get->key),
+                 ntohl(get->xquery_size), xquery);
     GNUNET_free (tmp);
   }
 
@@ -1855,8 +2114,16 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
         GNUNET_CONTAINER_bloomfilter_init (&xquery[xquery_size], reply_bf_size,
                                            GNUNET_CONSTANTS_BLOOMFILTER_K);
   eval =
-      GNUNET_BLOCK_evaluate (GDS_block_context, type, &get->key, &reply_bf,
-                             get->bf_mutator, xquery, xquery_size, NULL, 0);
+      GNUNET_BLOCK_evaluate (GDS_block_context,
+                             type,
+                             GNUNET_BLOCK_EO_NONE,
+                             &get->key,
+                             &reply_bf,
+                             get->bf_mutator,
+                             xquery,
+                             xquery_size,
+                             NULL,
+                             0);
   if (eval != GNUNET_BLOCK_EVALUATION_REQUEST_VALID)
   {
     /* request invalid or block type not supported */
@@ -1870,12 +2137,14 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
                                          GNUNET_CONSTANTS_BLOOMFILTER_K);
   GNUNET_break_op (GNUNET_YES ==
                    GNUNET_CONTAINER_bloomfilter_test (peer_bf,
-                                                      &peer->hashPubKey));
+                                                      &phash));
   /* remember request for routing replies */
   GDS_ROUTING_add (peer, type, options, &get->key, xquery, xquery_size,
                    reply_bf, get->bf_mutator);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "GET for %s at %s after %u hops\n",
-              GNUNET_h2s (&get->key), GNUNET_i2s (&my_identity),
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "GET for %s at %s after %u hops\n",
+              GNUNET_h2s (&get->key),
+              GNUNET_i2s (&my_identity),
               (unsigned int) ntohl (get->hop_count));
   /* local lookup (this may update the reply_bf) */
   if ((0 != (options & GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE)) ||
@@ -1903,19 +2172,27 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
                               1, GNUNET_NO);
   }
 
-  GDS_CLIENTS_process_get (options,
+  /* P2P forwarding */
+  forwarded = GNUNET_NO;
+  if (eval != GNUNET_BLOCK_EVALUATION_OK_LAST)
+    forwarded = GDS_NEIGHBOURS_handle_get (type, options,
+                                           ntohl (get->desired_replication_level),
+                                           ntohl (get->hop_count),
+                                           &get->key,
+                                           xquery,
+                                           xquery_size,
+                                           reply_bf,
+                                           get->bf_mutator, peer_bf);
+  GDS_CLIENTS_process_get (options
+                           | (GNUNET_OK == forwarded)
+                           ? GNUNET_DHT_RO_LAST_HOP : 0,
                            type,
-                           ntohl(get->hop_count),
-                           ntohl(get->desired_replication_level),
+                           ntohl (get->hop_count),
+                           ntohl (get->desired_replication_level),
                            0, NULL,
                            &get->key);
 
-  /* P2P forwarding */
-  if (eval != GNUNET_BLOCK_EVALUATION_OK_LAST)
-    GDS_NEIGHBOURS_handle_get (type, options,
-                               ntohl (get->desired_replication_level),
-                               ntohl (get->hop_count), &get->key, xquery,
-                               xquery_size, reply_bf, get->bf_mutator, peer_bf);
+
   /* clean up */
   if (NULL != reply_bf)
     GNUNET_CONTAINER_bloomfilter_free (reply_bf);
@@ -1930,10 +2207,11 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer,
  * @param cls closure
  * @param message message
  * @param peer peer identity this notification is about
- * @return GNUNET_YES (do not cut p2p connection)
+ * @return #GNUNET_YES (do not cut p2p connection)
  */
 static int
-handle_dht_p2p_result (void *cls, const struct GNUNET_PeerIdentity *peer,
+handle_dht_p2p_result (void *cls,
+                       const struct GNUNET_PeerIdentity *peer,
                        const struct GNUNET_MessageHeader *message)
 {
   const struct PeerResultMessage *prm;
@@ -1986,17 +2264,19 @@ handle_dht_p2p_result (void *cls, const struct GNUNET_PeerIdentity *peer,
     char *tmp;
 
     tmp = GNUNET_strdup (GNUNET_i2s (&my_identity));
-    LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG, "XDHT RESULT %s: %s->%s (%u)\n", 
-                 GNUNET_h2s (&prm->key), GNUNET_i2s (peer), tmp,
+    LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
+                 "R5N RESULT %s: %s->%s (%u)\n",
+                 GNUNET_h2s (&prm->key),
+                 GNUNET_i2s (peer),
+                 tmp,
                  get_path_length + 1);
     GNUNET_free (tmp);
   }
   /* if we got a HELLO, consider it for our own routing table */
-  if (type == GNUNET_BLOCK_TYPE_DHT_HELLO)
+  if (GNUNET_BLOCK_TYPE_DHT_HELLO == type)
   {
     const struct GNUNET_MessageHeader *h;
     struct GNUNET_PeerIdentity pid;
-    int bucket;
 
     /* Should be a HELLO, validate and consider using it! */
     if (data_size < sizeof (struct GNUNET_MessageHeader))
@@ -2011,51 +2291,75 @@ handle_dht_p2p_result (void *cls, const struct GNUNET_PeerIdentity *peer,
       return GNUNET_YES;
     }
     if (GNUNET_OK !=
-        GNUNET_HELLO_get_id ((const struct GNUNET_HELLO_Message *) h, &pid))
+        GNUNET_HELLO_get_id ((const struct GNUNET_HELLO_Message *) h,
+                             &pid))
     {
       GNUNET_break_op (0);
       return GNUNET_YES;
     }
-    if ((GNUNET_YES != disable_try_connect) &&
-        0 != memcmp (&my_identity, &pid, sizeof (struct GNUNET_PeerIdentity)))
-    {
-      bucket = find_bucket (&pid.hashPubKey);
-      if ((bucket >= 0) &&
-          (k_buckets[bucket].peers_size < bucket_size) &&
-          (NULL != GDS_transport_handle))
-      {
-        GNUNET_TRANSPORT_offer_hello (GDS_transport_handle, h, NULL, NULL);
-        GNUNET_TRANSPORT_try_connect (GDS_transport_handle, &pid, NULL, NULL); /*FIXME TRY_CONNECT change */
-      }
-    }
+    if ( (GNUNET_YES != disable_try_connect) &&
+         (0 != memcmp (&my_identity,
+                       &pid,
+                       sizeof (struct GNUNET_PeerIdentity))) )
+      try_connect (&pid,
+                   h);
   }
 
   /* append 'peer' to 'get_path' */
   {
     struct GNUNET_PeerIdentity xget_path[get_path_length + 1];
 
-    memcpy (xget_path, get_path,
+    memcpy (xget_path,
+            get_path,
             get_path_length * sizeof (struct GNUNET_PeerIdentity));
     xget_path[get_path_length] = *peer;
     get_path_length++;
 
     /* forward to local clients */
     GDS_CLIENTS_handle_reply (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                              &prm->key, get_path_length, xget_path,
-                              put_path_length, put_path, type, data_size, data);
+                              &prm->key,
+                              get_path_length,
+                              xget_path,
+                              put_path_length,
+                              put_path,
+                              type,
+                              data_size,
+                              data);
     GDS_CLIENTS_process_get_resp (type,
-                                  xget_path, get_path_length,
+                                  xget_path,
+                                  get_path_length,
                                   put_path, put_path_length,
-                                  GNUNET_TIME_absolute_ntoh (
-                                    prm->expiration_time),
+                                  GNUNET_TIME_absolute_ntoh (prm->expiration_time),
                                   &prm->key,
                                   data,
                                   data_size);
-
+    if (GNUNET_YES == cache_results)
+    {
+      struct GNUNET_PeerIdentity xput_path[get_path_length + 1 + put_path_length];
+
+      memcpy (xput_path, put_path, put_path_length * sizeof (struct GNUNET_PeerIdentity));
+      memcpy (&xput_path[put_path_length],
+             xget_path,
+             get_path_length * sizeof (struct GNUNET_PeerIdentity));
+
+      GDS_DATACACHE_handle_put (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+                               &prm->key,
+                               get_path_length + put_path_length,
+                                xput_path,
+                               type,
+                                data_size,
+                                data);
+    }
     /* forward to other peers */
-    GDS_ROUTING_process (type, GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                         &prm->key, put_path_length, put_path, get_path_length,
-                         xget_path, data, data_size);
+    GDS_ROUTING_process (type,
+                         GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+                         &prm->key,
+                         put_path_length,
+                         put_path,
+                         get_path_length,
+                         xget_path,
+                         data,
+                         data_size);
   }
 
   return GNUNET_YES;
@@ -2065,7 +2369,7 @@ handle_dht_p2p_result (void *cls, const struct GNUNET_PeerIdentity *peer,
 /**
  * Initialize neighbours subsystem.
  *
- * @return GNUNET_OK on success, GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 int
 GDS_NEIGHBOURS_init ()
@@ -2084,16 +2388,26 @@ GDS_NEIGHBOURS_init ()
       GNUNET_CONFIGURATION_get_value_number (GDS_cfg, "DHT", "bucket_size",
                                              &temp_config_num))
     bucket_size = (unsigned int) temp_config_num;
+  cache_results
+    = GNUNET_CONFIGURATION_get_value_yesno (GDS_cfg, "DHT", "CACHE_RESULTS");
+
   log_route_details_stderr =
     (NULL != getenv("GNUNET_DHT_ROUTE_DEBUG")) ? GNUNET_YES : GNUNET_NO;
-  atsAPI = GNUNET_ATS_performance_init (GDS_cfg, NULL, NULL);
-  coreAPI =
-      GNUNET_CORE_connect (GDS_cfg, NULL, &core_init, &handle_core_connect,
-                           &handle_core_disconnect, NULL, GNUNET_NO, NULL,
-                           GNUNET_NO, core_handlers);
-  if (coreAPI == NULL)
+  ats_ch = GNUNET_ATS_connectivity_init (GDS_cfg);
+  core_api =
+      GNUNET_CORE_connect (GDS_cfg, NULL,
+                           &core_init,
+                           &handle_core_connect,
+                           &handle_core_disconnect,
+                           NULL, GNUNET_NO,
+                           NULL, GNUNET_NO,
+                           core_handlers);
+  if (core_api == NULL)
     return GNUNET_SYSERR;
-  all_known_peers = GNUNET_CONTAINER_multihashmap_create (256, GNUNET_NO);
+  all_connected_peers = GNUNET_CONTAINER_multipeermap_create (256,
+                                                              GNUNET_NO);
+  all_desired_peers = GNUNET_CONTAINER_multipeermap_create (256,
+                                                            GNUNET_NO);
   return GNUNET_OK;
 }
 
@@ -2104,31 +2418,37 @@ GDS_NEIGHBOURS_init ()
 void
 GDS_NEIGHBOURS_done ()
 {
-  if (coreAPI == NULL)
+  if (NULL == core_api)
     return;
-  GNUNET_CORE_disconnect (coreAPI);
-  coreAPI = NULL;
-  GNUNET_ATS_performance_done (atsAPI);
-  atsAPI = NULL;
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (all_known_peers));
-  GNUNET_CONTAINER_multihashmap_destroy (all_known_peers);
-  all_known_peers = NULL;
-  if (GNUNET_SCHEDULER_NO_TASK != find_peer_task)
+  GNUNET_CORE_disconnect (core_api);
+  core_api = NULL;
+  GNUNET_assert (0 == GNUNET_CONTAINER_multipeermap_size (all_connected_peers));
+  GNUNET_CONTAINER_multipeermap_destroy (all_connected_peers);
+  all_connected_peers = NULL;
+  GNUNET_CONTAINER_multipeermap_iterate (all_desired_peers,
+                                         &free_connect_info,
+                                         NULL);
+  GNUNET_CONTAINER_multipeermap_destroy (all_desired_peers);
+  all_desired_peers = NULL;
+  GNUNET_ATS_connectivity_done (ats_ch);
+  ats_ch = NULL;
+  if (NULL != find_peer_task)
   {
     GNUNET_SCHEDULER_cancel (find_peer_task);
-    find_peer_task = GNUNET_SCHEDULER_NO_TASK;
+    find_peer_task = NULL;
   }
 }
 
+
 /**
  * Get the ID of the local node.
- * 
+ *
  * @return identity of the local node
  */
 struct GNUNET_PeerIdentity *
 GDS_NEIGHBOURS_get_id ()
 {
-    return &my_identity;
+  return &my_identity;
 }