dhtlog updates
[oweals/gnunet.git] / src / dv / gnunet-service-dv.c
index 3e47e17fc964f9bbb5599989c2da1cea643aab71..6c7795302a54f2c3feb87effc3fff80133073aea 100644 (file)
@@ -4,7 +4,7 @@
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
@@ -43,6 +43,7 @@
 #include "gnunet_hello_lib.h"
 #include "gnunet_peerinfo_service.h"
 #include "gnunet_crypto_lib.h"
+#include "gnunet_statistics_service.h"
 #include "dv.h"
 
 /**
 
 #define USE_PEER_ID GNUNET_YES
 
+/**
+ * How many outstanding messages (unknown sender) will we allow per peer?
+ */
+#define MAX_OUTSTANDING_MESSAGES 5
+
 /**
  * How often do we check about sending out more peer information (if
  * we are connected to no peers previously).
@@ -242,6 +248,38 @@ struct NeighborUpdateInfo
 
 };
 
+/**
+ * Struct to store a single message received with
+ * an unknown sender.
+ */
+struct UnknownSenderMessage
+{
+  /**
+   * Message sender (immediate)
+   */
+  struct GNUNET_PeerIdentity sender;
+
+  /**
+   * The actual message received
+   */
+  struct GNUNET_MessageHeader *message;
+
+  /**
+   * Latency of connection
+   */
+  struct GNUNET_TIME_Relative latency;
+
+  /**
+   * Distance to destination
+   */
+  uint32_t distance;
+
+  /**
+   * Unknown sender id
+   */
+  uint32_t sender_id;
+};
+
 /**
  * Struct where actual neighbor information is stored,
  * referenced by min_heap and max_heap.  Freeing dealt
@@ -279,6 +317,13 @@ struct DirectNeighbor
    * from DV?
    */
   int hidden;
+
+  /**
+   * Save messages immediately from this direct neighbor from a
+   * distan peer we don't know on the chance that it will be
+   * gossiped about and we can deliver the message.
+   */
+  struct UnknownSenderMessage pending_messages[MAX_OUTSTANDING_MESSAGES];
 };
 
 
@@ -329,6 +374,11 @@ struct DistantNeighbor
    */
   struct GNUNET_TIME_Absolute last_activity;
 
+  /**
+   * Last time we sent routing information about this peer
+   */
+  struct GNUNET_TIME_Absolute last_gossip;
+
   /**
    * Cost to neighbor, used for actual distance vector computations
    */
@@ -598,6 +648,11 @@ static struct GNUNET_CONTAINER_Heap *neighbor_min_heap;
  */
 static struct GNUNET_CONTAINER_Heap *neighbor_max_heap;
 
+/**
+ * Handle for the statistics service.
+ */
+struct GNUNET_STATISTICS_Handle *stats;
+
 /**
  * How far out to keep peers we learn about.
  */
@@ -693,8 +748,8 @@ size_t transmit_to_plugin (void *cls,
   if (buf == NULL)
     {
       /* client disconnected */
-#if DEBUG_DV
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "`%s': buffer was NULL\n", "DHT");
+#if DEBUG_DV_MESSAGES
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%s: %s buffer was NULL (client disconnect?)\n", my_short_id, "transmit_to_plugin");
 #endif
       return 0;
     }
@@ -703,9 +758,6 @@ size_t transmit_to_plugin (void *cls,
   while ( (NULL != (reply = plugin_pending_head)) &&
           (size >= off + (msize = ntohs (reply->msg->size))))
     {
-#if DEBUG_DV
-    GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "`%s' : transmit_notify (plugin) called with size %d\n", "dv service", msize);
-#endif
       GNUNET_CONTAINER_DLL_remove (plugin_pending_head,
                                    plugin_pending_tail,
                                    reply);
@@ -741,11 +793,6 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
 {
   struct GNUNET_DV_MessageReceived *received_msg;
   struct PendingMessage *pending_message;
-#if DEBUG_DV_MESSAGES
-  struct GNUNET_MessageHeader * packed_message_header;
-  struct GNUNET_HELLO_Message *hello_msg;
-  struct GNUNET_PeerIdentity hello_identity;
-#endif
   char *sender_address;
   size_t sender_address_len;
   char *packed_msg_start;
@@ -773,7 +820,6 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
   received_msg = GNUNET_malloc(size);
   received_msg->header.size = htons(size);
   received_msg->header.type = htons(GNUNET_MESSAGE_TYPE_TRANSPORT_DV_RECEIVE);
-  received_msg->sender_address_len = htonl(sender_address_len);
   received_msg->distance = htonl(cost);
   received_msg->msg_len = htonl(message_size);
   /* Set the sender in this message to be the original sender! */
@@ -785,15 +831,6 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
   packed_msg_start = (char *)&received_msg[1];
   packed_msg_start = &packed_msg_start[sender_address_len];
   memcpy(packed_msg_start, message, message_size);
-#if DEBUG_DV_MESSAGES
-  packed_message_header = (struct GNUNET_MessageHeader *)packed_msg_start;
-  if (ntohs(packed_message_header->type) == GNUNET_MESSAGE_TYPE_HELLO)
-    {
-      hello_msg = (struct GNUNET_HELLO_Message *)packed_msg_start;
-      GNUNET_assert(GNUNET_OK == GNUNET_HELLO_get_id(hello_msg, &hello_identity));
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%s: send_to_plugin: Packed HELLO message is about peer %s\n", my_short_id, GNUNET_i2s(&hello_identity));
-    }
-#endif
   pending_message = GNUNET_malloc(sizeof(struct PendingMessage) + size);
   pending_message->msg = (struct GNUNET_MessageHeader *)&pending_message[1];
   memcpy(&pending_message[1], received_msg, size);
@@ -809,12 +846,6 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
                                                                         size, GNUNET_TIME_UNIT_FOREVER_REL,
                                                                         &transmit_to_plugin, NULL);
         }
-#if DEBUG_DV_MESSAGES
-      else
-        {
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to queue message for plugin, must be one in progress already!!\n");
-        }
-#endif
     }
   else
     {
@@ -822,6 +853,26 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
     }
 }
 
+/* Declare here so retry_core_send is aware of it */
+size_t core_transmit_notify (void *cls,
+                             size_t size, void *buf);
+
+/**
+ *  Try to send another message from our core sending list
+ */
+static void
+try_core_send (void *cls,
+                 const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PendingMessage *pending;
+  pending = core_pending_head;
+
+  if (core_transmit_handle != NULL)
+    return; /* Message send already in progress */
+
+  if (pending != NULL)
+    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, pending->importance, pending->timeout, &pending->recipient, pending->msg_size, &core_transmit_notify, NULL);
+}
 
 /**
  * Function called to notify a client about the socket
@@ -829,7 +880,7 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
  * NULL and "size" zero if the socket was closed for
  * writing in the meantime.
  *
- * @param cls closure
+ * @param cls closure (NULL)
  * @param size number of bytes available in buf
  * @param buf where the callee should write the message
  * @return number of bytes written to buf
@@ -838,7 +889,7 @@ size_t core_transmit_notify (void *cls,
                              size_t size, void *buf)
 {
   char *cbuf = buf;
-  struct PendingMessage *reply;
+  struct PendingMessage *pending;
   struct PendingMessage *client_reply;
   size_t off;
   size_t msize;
@@ -854,22 +905,22 @@ size_t core_transmit_notify (void *cls,
 
   core_transmit_handle = NULL;
   off = 0;
-  reply = core_pending_head;
-  if ( (reply != NULL) &&
-          (size >= (msize = ntohs (reply->msg->size))))
+  pending = core_pending_head;
+  if ( (pending != NULL) &&
+          (size >= (msize = ntohs (pending->msg->size))))
     {
 #if DEBUG_DV
       GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "`%s' : transmit_notify (core) called with size %d\n", "dv service", msize);
 #endif
       GNUNET_CONTAINER_DLL_remove (core_pending_head,
                                    core_pending_tail,
-                                   reply);
-      if (reply->send_result != NULL) /* Will only be non-null if a real client asked for this send */
+                                   pending);
+      if (pending->send_result != NULL) /* Will only be non-null if a real client asked for this send */
         {
           client_reply = GNUNET_malloc(sizeof(struct PendingMessage) + sizeof(struct GNUNET_DV_SendResultMessage));
           client_reply->msg = (struct GNUNET_MessageHeader *)&client_reply[1];
-          memcpy(&client_reply[1], reply->send_result, sizeof(struct GNUNET_DV_SendResultMessage));
-          GNUNET_free(reply->send_result);
+          memcpy(&client_reply[1], pending->send_result, sizeof(struct GNUNET_DV_SendResultMessage));
+          GNUNET_free(pending->send_result);
 
           GNUNET_CONTAINER_DLL_insert_after(plugin_pending_head, plugin_pending_tail, plugin_pending_tail, client_reply);
           if (client_handle != NULL)
@@ -887,13 +938,15 @@ size_t core_transmit_notify (void *cls,
                 }
             }
         }
-      memcpy (&cbuf[off], reply->msg, msize);
-      GNUNET_free (reply);
+      memcpy (&cbuf[off], pending->msg, msize);
+      GNUNET_free (pending);
       off += msize;
     }
-  reply = core_pending_head;
-  if (reply != NULL)
-    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, reply->importance, reply->timeout, &reply->recipient, reply->msg_size, &core_transmit_notify, NULL);
+  /*reply = core_pending_head;*/
+
+  GNUNET_SCHEDULER_add_now(sched, &try_core_send, NULL);
+  /*if (reply != NULL)
+    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, reply->importance, reply->timeout, &reply->recipient, reply->msg_size, &core_transmit_notify, NULL);*/
 
   return off;
 }
@@ -990,14 +1043,8 @@ send_message_via (const struct GNUNET_PeerIdentity *sender,
                                      core_pending_tail,
                                      pending_message);
 
-  if (core_transmit_handle == NULL)
-    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, send_context->importance, send_context->timeout, recipient, msg_size, &core_transmit_notify, NULL);
-  else
-    {
-#if DEBUG_DV
-      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "`%s': Failed to schedule pending transmission (must be one in progress!)\n", "dv service");
-#endif
-    }
+  GNUNET_SCHEDULER_add_now(sched, try_core_send, NULL);
+
   return GNUNET_YES;
 }
 
@@ -1037,6 +1084,7 @@ find_least_cost_peer (void *cls,
  * @param message the packed message
  * @param message_size size of the message
  * @param importance what priority to send this message with
+ * @param uid the unique identifier of this message (or 0 for none)
  * @param timeout how long to possibly delay sending this message
  */
 static int
@@ -1085,7 +1133,7 @@ send_message (const struct GNUNET_PeerIdentity * recipient,
   recipient_id = target->referrer_id;
 
   source = GNUNET_CONTAINER_multihashmap_get (extended_neighbors,
-                                      &sender->hashPubKey);
+                                              &sender->hashPubKey);
   if (source == NULL)
     {
       if (0 != (memcmp (&my_identity,
@@ -1139,6 +1187,10 @@ send_message (const struct GNUNET_PeerIdentity * recipient,
   GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Sending DATA message. Sender id %u, source %s, destination %s, via %s\n", GNUNET_i2s(&my_identity), sender_id, &encPeerFrom, &encPeerTo, &encPeerVia);
 #endif
   memcpy (&toSend[1], message, message_size);
+  if ((source != NULL) && (source->pkey == NULL)) /* Test our hypothesis about message failures! */
+    {
+      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Sending message, but anticipate recipient will not know sender!!!\n\n\n", my_short_id);
+    }
   GNUNET_CONTAINER_DLL_insert_after (core_pending_head,
                                      core_pending_tail,
                                      core_pending_tail,
@@ -1146,10 +1198,8 @@ send_message (const struct GNUNET_PeerIdentity * recipient,
 #if DEBUG_DV
   GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Notifying core of send size %d to destination `%s'\n", "DV SEND MESSAGE", msg_size, GNUNET_i2s(recipient));
 #endif
-  if (core_transmit_handle == NULL)
-    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, importance, timeout, &target->referrer->identity, msg_size, &core_transmit_notify, NULL);
-  else
-    GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: CORE ALREADY SENDING\n", "DV SEND MESSAGE", msg_size);
+
+  GNUNET_SCHEDULER_add_now(sched, try_core_send, NULL);
   return (int) cost;
 }
 
@@ -1222,6 +1272,37 @@ void tokenized_message_handler (void *cls,
   }
 }
 
+#if DELAY_FORWARDS
+struct DelayedMessageContext
+{
+  struct GNUNET_PeerIdentity dest;
+  struct GNUNET_PeerIdentity sender;
+  struct GNUNET_MessageHeader *message;
+  size_t message_size;
+  uint32_t uid;
+};
+
+void send_message_delayed (void *cls,
+                           const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct DelayedMessageContext *msg_ctx = cls;
+  if (msg_ctx != NULL)
+    {
+      send_message(&msg_ctx->dest,
+                   &msg_ctx->sender,
+                   NULL,
+                   msg_ctx->message,
+                   msg_ctx->message_size,
+                   default_dv_priority,
+                   msg_ctx->uid,
+                   GNUNET_TIME_relative_get_forever());
+      GNUNET_free(msg_ctx->message);
+      GNUNET_free(msg_ctx);
+    }
+}
+#endif
+
+
 /**
  * Core handler for dv data messages.  Whatever this message
  * contains all we really have to do is rip it out of its
@@ -1235,10 +1316,10 @@ void tokenized_message_handler (void *cls,
  * @param distance the distance to the immediate peer
  */
 static int handle_dv_data_message (void *cls,
-                             const struct GNUNET_PeerIdentity * peer,
-                             const struct GNUNET_MessageHeader * message,
-                             struct GNUNET_TIME_Relative latency,
-                             uint32_t distance)
+                                   const struct GNUNET_PeerIdentity * peer,
+                                   const struct GNUNET_MessageHeader * message,
+                                   struct GNUNET_TIME_Relative latency,
+                                   uint32_t distance)
 {
   const p2p_dv_MESSAGE_Data *incoming = (const p2p_dv_MESSAGE_Data *) message;
   const struct GNUNET_MessageHeader *packed_message;
@@ -1246,24 +1327,26 @@ static int handle_dv_data_message (void *cls,
   struct DistantNeighbor *pos;
   unsigned int sid;             /* Sender id */
   unsigned int tid;             /* Target id */
-  struct GNUNET_PeerIdentity original_sender;
-  struct GNUNET_PeerIdentity destination;
+  struct GNUNET_PeerIdentity *original_sender;
+  struct GNUNET_PeerIdentity *destination;
   struct FindDestinationContext fdc;
   struct TokenizedMessageContext tkm_ctx;
+  int i;
+  int found_pos;
+#if DELAY_FORWARDS
+  struct DelayedMessageContext *delayed_context;
+#endif
 #if USE_PEER_ID
   struct CheckPeerContext checkPeerCtx;
 #endif
+#if DEBUG_DV_MESSAGES
   char *sender_id;
-  char *direct_id;
+#endif
   int ret;
   size_t packed_message_size;
   char *cbuf;
-#if NO_MST
-  size_t offset;
-#endif
-  packed_message_size = ntohs(incoming->header.size) - sizeof(p2p_dv_MESSAGE_Data);
-
 
+  packed_message_size = ntohs(incoming->header.size) - sizeof(p2p_dv_MESSAGE_Data);
 #if DEBUG_DV
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "%s: Receives DATA message from %s size %d, packed size %d!\n", my_short_id, GNUNET_i2s(peer) , ntohs(incoming->header.size), packed_message_size);
@@ -1280,15 +1363,10 @@ static int handle_dv_data_message (void *cls,
     }
 
   dn = GNUNET_CONTAINER_multihashmap_get (direct_neighbors,
-                                  &peer->hashPubKey);
+                                          &peer->hashPubKey);
   if (dn == NULL)
-    {
-#if DEBUG_DV
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: dn NULL!\n", "dv");
-#endif
-      return GNUNET_OK;
-    }
+    return GNUNET_OK;
+
   sid = ntohl (incoming->sender);
 #if USE_PEER_ID
   if (sid != 0)
@@ -1311,12 +1389,9 @@ static int handle_dv_data_message (void *cls,
 
   if (pos == NULL)
     {
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
 #if DEBUG_DV_MESSAGES
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: unknown sender (%u), Message uid %llu from %s!\n", GNUNET_i2s(&my_identity), ntohl(incoming->sender), ntohl(incoming->uid), direct_id);
-#endif
-      GNUNET_free(direct_id);
+                  "%s: unknown sender (%u), Message uid %u from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
       pos = dn->referee_head;
       while ((NULL != pos) && (pos->referrer_id != sid))
       {
@@ -1325,17 +1400,38 @@ static int handle_dv_data_message (void *cls,
         GNUNET_free(sender_id);
         pos = pos->next;
       }
-
-#if DEBUG_MESSAGE_DROP
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: DROPPING MESSAGE type %d, unknown sender! Message immediately from %s!\n", GNUNET_i2s(&my_identity), ntohs(((struct GNUNET_MessageHeader *)&incoming[1])->type), direct_id);
-      GNUNET_free(direct_id);
 #endif
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "%s: unknown sender (%u), Message uid %u from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
+
+      found_pos = -1;
+      for (i = 0; i< MAX_OUTSTANDING_MESSAGES; i++)
+        {
+          if (dn->pending_messages[i].sender_id == 0)
+            {
+              found_pos = i;
+              break;
+            }
+        }
+
+      if (found_pos == -1)
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                      "%s: Too many unknown senders (%u), ignoring message! Message uid %llu from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
+        }
+      else
+        {
+            dn->pending_messages[found_pos].message = GNUNET_malloc(ntohs (message->size));
+            memcpy(dn->pending_messages[found_pos].message, message, ntohs(message->size));
+            dn->pending_messages[found_pos].distance = distance;
+            dn->pending_messages[found_pos].latency = latency;
+            memcpy(&dn->pending_messages[found_pos].sender, peer, sizeof(struct GNUNET_PeerIdentity));
+            dn->pending_messages[found_pos].sender_id = sid;
+        }
       /* unknown sender */
       return GNUNET_OK;
     }
-  original_sender = pos->identity;
+  original_sender = &pos->identity;
   tid = ntohl (incoming->recipient);
   if (tid == 0)
     {
@@ -1355,27 +1451,6 @@ static int handle_dv_data_message (void *cls,
           GNUNET_break_op(0);
           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: %s Received corrupt data, discarding!", my_short_id, "DV SERVICE");
         }
-#if NO_MST
-      offset = 0;
-      while(offset < packed_message_size)
-        {
-          packed_message = (struct GNUNET_MessageHeader *)&cbuf[offset];
-
-          GNUNET_break_op (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_GOSSIP);
-          GNUNET_break_op (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_DATA);
-          if ( (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_GOSSIP) &&
-              (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_DATA) )
-          {
-#if DEBUG_DV_MESSAGES
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                        "%s: Receives %s message(s) for me, uid %u, total size %d cost %u from %s!\n", my_short_id, "DV DATA", ntohl(incoming->uid), ntohs(packed_message->size), pos->cost, GNUNET_i2s(&pos->identity));
-#endif
-            GNUNET_assert(memcmp(peer, &pos->identity, sizeof(struct GNUNET_PeerIdentity)) != 0);
-            send_to_plugin(peer, packed_message, ntohs(packed_message->size), &pos->identity, pos->cost);
-          }
-          offset += ntohs(packed_message->size);
-        }
-#endif
       return GNUNET_OK;
     }
   else
@@ -1403,11 +1478,11 @@ static int handle_dv_data_message (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "%s: Receives %s message uid %u for someone we don't know (id %u)!\n", my_short_id, "DV DATA", ntohl(incoming->uid), tid);
 #endif
-    return GNUNET_OK;
+      return GNUNET_OK;
     }
-  destination = fdc.dest->identity;
+  destination = &fdc.dest->identity;
 
-  if (0 == memcmp (&destination, peer, sizeof (struct GNUNET_PeerIdentity)))
+  if (0 == memcmp (destination, peer, sizeof (struct GNUNET_PeerIdentity)))
     {
       /* FIXME: create stat: routing loop-discard! */
 #if DEBUG_DV_PEER_NUMBERS
@@ -1415,9 +1490,8 @@ static int handle_dv_data_message (void *cls,
 #endif
 
 #if DEBUG_DV_MESSAGES
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: DROPPING MESSAGE uid %u type %d, routing loop! Message immediately from %s!\n", my_short_id, ntohl(incoming->uid), ntohs(packed_message->type), direct_id);
+                  "%s: DROPPING MESSAGE uid %u type %d, routing loop! Message immediately from %s!\n", my_short_id, ntohl(incoming->uid), ntohs(packed_message->type), GNUNET_i2s(&dn->identity));
 #endif
       return GNUNET_OK;
     }
@@ -1425,27 +1499,36 @@ static int handle_dv_data_message (void *cls,
   /* At this point we have a message, and we need to forward it on to the
    * next DV hop.
    */
-  /* FIXME: Can't send message on, we have to behave.
-   * We have to tell core we have a message for the next peer, and let
-   * transport do transport selection on how to get this message to 'em */
-  /*ret = send_message (&destination,
-                      &original_sender,
-                      packed_message, DV_PRIORITY, DV_DELAY);*/
-
 #if DEBUG_DV_MESSAGES
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "%s: FORWARD %s message for %s, uid %u, size %d type %d, cost %u!\n", my_short_id, "DV DATA", GNUNET_i2s(&destination), ntohl(incoming->uid), ntohs(packed_message->size), ntohs(packed_message->type), pos->cost);
+              "%s: FORWARD %s message for %s, uid %u, size %d type %d, cost %u!\n", my_short_id, "DV DATA", GNUNET_i2s(destination), ntohl(incoming->uid), ntohs(packed_message->size), ntohs(packed_message->type), pos->cost);
 #endif
 
-  ret = send_message(&destination,
-                     &original_sender,
-                     NULL,
-                     packed_message,
-                     packed_message_size,
-                     default_dv_priority,
-                     ntohl(incoming->uid),
-                     GNUNET_TIME_relative_get_forever());
-
+#if DELAY_FORWARDS
+  if (GNUNET_TIME_absolute_get_duration(pos->last_gossip).value < GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2).value)
+    {
+      delayed_context = GNUNET_malloc(sizeof(struct DelayedMessageContext));
+      memcpy(&delayed_context->dest, destination, sizeof(struct GNUNET_PeerIdentity));
+      memcpy(&delayed_context->sender, original_sender, sizeof(struct GNUNET_PeerIdentity));
+      delayed_context->message = GNUNET_malloc(packed_message_size);
+      memcpy(delayed_context->message, packed_message, packed_message_size);
+      delayed_context->message_size = packed_message_size;
+      delayed_context->uid = ntohl(incoming->uid);
+      GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 2500), &send_message_delayed, delayed_context);
+      return GNUNET_OK;
+    }
+  else
+#endif
+    {
+      ret = send_message(destination,
+                         original_sender,
+                         NULL,
+                         packed_message,
+                         packed_message_size,
+                         default_dv_priority,
+                         ntohl(incoming->uid),
+                         GNUNET_TIME_relative_get_forever());
+    }
   if (ret != GNUNET_SYSERR)
     return GNUNET_OK;
   else
@@ -1563,6 +1646,7 @@ neighbor_send_task (void *cls,
       GNUNET_free(encPeerAbout);
       GNUNET_free(encPeerTo);
 #endif
+      about->last_gossip = GNUNET_TIME_absolute_get();
       pending_message = GNUNET_malloc(sizeof(struct PendingMessage) + sizeof(p2p_dv_MESSAGE_NeighborInfo));
       pending_message->msg = (struct GNUNET_MessageHeader *)&pending_message[1];
       pending_message->importance = default_dv_priority;
@@ -1584,8 +1668,9 @@ neighbor_send_task (void *cls,
                                          core_pending_tail,
                                          pending_message);
 
-      if (core_transmit_handle == NULL)
-        core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, default_dv_priority, GNUNET_TIME_relative_get_forever(), &to->identity, sizeof(p2p_dv_MESSAGE_NeighborInfo), &core_transmit_notify, NULL);
+      GNUNET_SCHEDULER_add_now(sched, try_core_send, NULL);
+      /*if (core_transmit_handle == NULL)
+        core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, default_dv_priority, GNUNET_TIME_relative_get_forever(), &to->identity, sizeof(p2p_dv_MESSAGE_NeighborInfo), &core_transmit_notify, NULL);*/
 
     }
 
@@ -1961,8 +2046,9 @@ static int schedule_disconnect_messages (void *cls,
                                      core_pending_tail,
                                      pending_message);
 
-  if (core_transmit_handle == NULL)
-    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, default_dv_priority, GNUNET_TIME_relative_get_forever(), &notify->identity, sizeof(p2p_dv_MESSAGE_Disconnect), &core_transmit_notify, NULL);
+  GNUNET_SCHEDULER_add_now(sched, try_core_send, NULL);
+  /*if (core_transmit_handle == NULL)
+    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, default_dv_priority, GNUNET_TIME_relative_get_forever(), &notify->identity, sizeof(p2p_dv_MESSAGE_Disconnect), &core_transmit_notify, NULL);*/
 
   return GNUNET_YES;
 }
@@ -2257,6 +2343,7 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
   unsigned int our_id;
   char *addr1;
   char *addr2;
+  int i;
 
 #if DEBUG_DV_PEER_NUMBERS
   char *encAbout;
@@ -2373,8 +2460,22 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
       GNUNET_CONTAINER_multihashmap_put (extended_neighbors, &peer->hashPubKey,
                                  neighbor,
                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-      if (cost != DIRECT_NEIGHBOR_COST)
+      if (referrer_peer_id != 0)
+        {
+          for (i = 0; i< MAX_OUTSTANDING_MESSAGES; i++)
+            {
+              if (referrer->pending_messages[i].sender_id == referrer_peer_id) /* We have a queued message from just learned about peer! */
+                {
+#if DEBUG_DV_MESSAGES
+                  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: learned about peer %llu from which we have a previous unknown message, processing!\n", my_short_id, referrer_peer_id);
+#endif
+                  handle_dv_data_message(NULL, &referrer->pending_messages[i].sender, referrer->pending_messages[i].message, referrer->pending_messages[i].latency, referrer->pending_messages[i].distance);
+                  GNUNET_free(referrer->pending_messages[i].message);
+                  referrer->pending_messages[i].sender_id = 0;
+                }
+            }
+        }
+      if ((cost != DIRECT_NEIGHBOR_COST) && (neighbor->pkey != NULL))
         {
           /* Added neighbor, now send HELLO to transport */
           hello_context = GNUNET_malloc(sizeof(struct HelloContext));
@@ -2385,7 +2486,9 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
           GNUNET_assert(memcmp(hello_context->direct_peer, &hello_context->distant_peer, sizeof(struct GNUNET_PeerIdentity)) != 0);
           addr1 = GNUNET_strdup(GNUNET_i2s(hello_context->direct_peer));
           addr2 = GNUNET_strdup(GNUNET_i2s(&hello_context->distant_peer));
+#if DEBUG_DV
           GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: GIVING HELLO size %d for %s via %s to TRANSPORT\n", my_short_id, GNUNET_HELLO_size(hello_msg), addr2, addr1);
+#endif
           GNUNET_free(addr1);
           GNUNET_free(addr2);
           send_to_plugin(hello_context->direct_peer, GNUNET_HELLO_get_header(hello_msg), GNUNET_HELLO_size(hello_msg), &hello_context->distant_peer, cost);
@@ -2450,8 +2553,10 @@ static int handle_dv_disconnect_message (void *cls,
       if (distant->referrer_id == ntohl(enc_message->peer_id))
         {
           distant_neighbor_free(distant);
+          distant = referrer->referee_head;
         }
-      distant = referrer->referee_head;
+      else
+        distant = distant->next;
     }
 
   return GNUNET_OK;
@@ -2658,12 +2763,11 @@ static int add_all_direct_neighbors (void *cls,
  * @param cls closure
  * @param peer id of the peer, NULL for last call
  * @param hello hello message for the peer (can be NULL)
- * @param trust amount of trust we have in the peer
  */
 static void
 process_peerinfo (void *cls,
                   const struct GNUNET_PeerIdentity *peer,
-                  const struct GNUNET_HELLO_Message *hello, uint32_t trust)
+                  const struct GNUNET_HELLO_Message *hello)
 {
   struct PeerIteratorContext *peerinfo_iterator = cls;
   struct DirectNeighbor *neighbor = peerinfo_iterator->neighbor;
@@ -2673,16 +2777,15 @@ process_peerinfo (void *cls,
 #endif
   int sent;
 
-  if (peer == NULL) /* && (neighbor->pkey == NULL))*/
+  if (peer == NULL)
     {
-      if (distant->pkey == NULL) /* FIXME: Reschedule? */
+      if (distant->pkey == NULL)
         {
 #if DEBUG_DV
           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to get peerinfo information for this peer, retrying!\n");
 #endif
           peerinfo_iterator->ic = GNUNET_PEERINFO_iterate(peerinfo_handle,
                                                           &peerinfo_iterator->neighbor->identity,
-                                                          0,
                                                           GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3),
                                                           &process_peerinfo,
                                                           peerinfo_iterator);
@@ -2705,19 +2808,20 @@ process_peerinfo (void *cls,
           memcpy(distant->pkey, &neighbor->pkey, sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded));
         }
 
-      /* Why do it this way, now we have the distant neighbor! */
-      /*GNUNET_CONTAINER_multihashmap_get_multiple(extended_neighbors,
-                                                 &peer->hashPubKey,
-                                                 &add_pkey_to_extended,
-                                                 &neighbor->pkey);*/
-
       sent = GNUNET_CONTAINER_multihashmap_iterate (extended_neighbors, &add_all_extended_peers, neighbor->send_context);
-
+      if (stats != NULL)
+        {
+          GNUNET_STATISTICS_update (stats, "# distant peers gossiped to direct neighbors", sent, GNUNET_NO);
+        }
 #if DEBUG_DV_PEER_NUMBERS
       neighbor_pid = GNUNET_strdup(GNUNET_i2s(&neighbor->identity));
       GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Gossipped %d extended peers to %s\n", GNUNET_i2s(&my_identity), sent, neighbor_pid);
 #endif
       sent = GNUNET_CONTAINER_multihashmap_iterate (direct_neighbors, &add_all_direct_neighbors, neighbor);
+      if (stats != NULL)
+        {
+          GNUNET_STATISTICS_update (stats, "# direct peers gossiped to direct neighbors", sent, GNUNET_NO);
+        }
 #if DEBUG_DV_PEER_NUMBERS
       GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Gossipped about %s to %d direct peers\n", GNUNET_i2s(&my_identity), neighbor_pid, sent);
       GNUNET_free(neighbor_pid);
@@ -2765,7 +2869,6 @@ void handle_core_connect (void *cls,
     peerinfo_iterator->neighbor = neighbor;
     peerinfo_iterator->ic = GNUNET_PEERINFO_iterate (peerinfo_handle,
                                                      peer,
-                                                     0,
                                                      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3),
                                                      &process_peerinfo,
                                                      peerinfo_iterator);
@@ -2785,7 +2888,13 @@ void handle_core_connect (void *cls,
   {
     about = GNUNET_CONTAINER_multihashmap_get(extended_neighbors, &peer->hashPubKey);
     if ((GNUNET_CONTAINER_multihashmap_get(direct_neighbors, &peer->hashPubKey) == NULL) && (about != NULL))
-      sent = GNUNET_CONTAINER_multihashmap_iterate(direct_neighbors, &add_distant_all_direct_neighbors, about);
+      {
+        sent = GNUNET_CONTAINER_multihashmap_iterate(direct_neighbors, &add_distant_all_direct_neighbors, about);
+        if (stats != NULL)
+          {
+            GNUNET_STATISTICS_update (stats, "# direct peers gossiped to new direct neighbors", sent, GNUNET_NO);
+          }
+      }
 #if DEBUG_DV
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "%s: Distance (%d) greater than %d or already know about peer (%s), not re-adding!\n", "dv", distance, DIRECT_NEIGHBOR_COST, GNUNET_i2s(peer));
@@ -2897,6 +3006,7 @@ run (void *cls,
                        &core_init,
                        &handle_core_connect,
                        &handle_core_disconnect,
+                      NULL,
                        NULL,
                        GNUNET_NO,
                        NULL,
@@ -2906,8 +3016,7 @@ run (void *cls,
   if (coreAPI == NULL)
     return;
 
-  coreMST = GNUNET_SERVER_mst_create (GNUNET_SERVER_MAX_MESSAGE_SIZE,
-                                      &tokenized_message_handler,
+  coreMST = GNUNET_SERVER_mst_create (&tokenized_message_handler,
                                       NULL);
 
    peerinfo_handle = GNUNET_PEERINFO_connect(sched, cfg);
@@ -2923,12 +3032,6 @@ run (void *cls,
                                 GNUNET_TIME_UNIT_FOREVER_REL,
                                 &shutdown_task,
                                 NULL);
-#if INSANE_GOSSIP
-  GNUNET_SCHEDULER_add_delayed (sched,
-                                GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 5),
-                                &gossip_all_to_all,
-                                NULL);
-#endif
 }