dhtlog updates
[oweals/gnunet.git] / src / dv / gnunet-service-dv.c
index 00de78aae345c023f31e726e345fca2e61664324..6c7795302a54f2c3feb87effc3fff80133073aea 100644 (file)
@@ -4,7 +4,7 @@
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
@@ -43,6 +43,7 @@
 #include "gnunet_hello_lib.h"
 #include "gnunet_peerinfo_service.h"
 #include "gnunet_crypto_lib.h"
+#include "gnunet_statistics_service.h"
 #include "dv.h"
 
 /**
 
 #define USE_PEER_ID GNUNET_YES
 
+/**
+ * How many outstanding messages (unknown sender) will we allow per peer?
+ */
+#define MAX_OUTSTANDING_MESSAGES 5
+
 /**
  * How often do we check about sending out more peer information (if
  * we are connected to no peers previously).
@@ -242,6 +248,38 @@ struct NeighborUpdateInfo
 
 };
 
+/**
+ * Struct to store a single message received with
+ * an unknown sender.
+ */
+struct UnknownSenderMessage
+{
+  /**
+   * Message sender (immediate)
+   */
+  struct GNUNET_PeerIdentity sender;
+
+  /**
+   * The actual message received
+   */
+  struct GNUNET_MessageHeader *message;
+
+  /**
+   * Latency of connection
+   */
+  struct GNUNET_TIME_Relative latency;
+
+  /**
+   * Distance to destination
+   */
+  uint32_t distance;
+
+  /**
+   * Unknown sender id
+   */
+  uint32_t sender_id;
+};
+
 /**
  * Struct where actual neighbor information is stored,
  * referenced by min_heap and max_heap.  Freeing dealt
@@ -279,6 +317,13 @@ struct DirectNeighbor
    * from DV?
    */
   int hidden;
+
+  /**
+   * Save messages immediately from this direct neighbor from a
+   * distan peer we don't know on the chance that it will be
+   * gossiped about and we can deliver the message.
+   */
+  struct UnknownSenderMessage pending_messages[MAX_OUTSTANDING_MESSAGES];
 };
 
 
@@ -603,6 +648,11 @@ static struct GNUNET_CONTAINER_Heap *neighbor_min_heap;
  */
 static struct GNUNET_CONTAINER_Heap *neighbor_max_heap;
 
+/**
+ * Handle for the statistics service.
+ */
+struct GNUNET_STATISTICS_Handle *stats;
+
 /**
  * How far out to keep peers we learn about.
  */
@@ -770,7 +820,6 @@ void send_to_plugin(const struct GNUNET_PeerIdentity * sender,
   received_msg = GNUNET_malloc(size);
   received_msg->header.size = htons(size);
   received_msg->header.type = htons(GNUNET_MESSAGE_TYPE_TRANSPORT_DV_RECEIVE);
-  received_msg->sender_address_len = htonl(sender_address_len);
   received_msg->distance = htonl(cost);
   received_msg->msg_len = htonl(message_size);
   /* Set the sender in this message to be the original sender! */
@@ -1035,6 +1084,7 @@ find_least_cost_peer (void *cls,
  * @param message the packed message
  * @param message_size size of the message
  * @param importance what priority to send this message with
+ * @param uid the unique identifier of this message (or 0 for none)
  * @param timeout how long to possibly delay sending this message
  */
 static int
@@ -1137,9 +1187,9 @@ send_message (const struct GNUNET_PeerIdentity * recipient,
   GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Sending DATA message. Sender id %u, source %s, destination %s, via %s\n", GNUNET_i2s(&my_identity), sender_id, &encPeerFrom, &encPeerTo, &encPeerVia);
 #endif
   memcpy (&toSend[1], message, message_size);
-  if (source->pkey == NULL) /* Test our hypothesis about message failures! */
+  if ((source != NULL) && (source->pkey == NULL)) /* Test our hypothesis about message failures! */
     {
-      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Sending message, but anticipate recipient will not know sender!!!\n\n\n");
+      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Sending message, but anticipate recipient will not know sender!!!\n\n\n", my_short_id);
     }
   GNUNET_CONTAINER_DLL_insert_after (core_pending_head,
                                      core_pending_tail,
@@ -1150,10 +1200,6 @@ send_message (const struct GNUNET_PeerIdentity * recipient,
 #endif
 
   GNUNET_SCHEDULER_add_now(sched, try_core_send, NULL);
-  /*if (core_transmit_handle == NULL)
-    core_transmit_handle = GNUNET_CORE_notify_transmit_ready(coreAPI, importance, timeout, &target->referrer->identity, msg_size, &core_transmit_notify, NULL);
-  else
-    GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: CORE ALREADY SENDING\n", "DV SEND MESSAGE", msg_size);*/
   return (int) cost;
 }
 
@@ -1226,6 +1272,7 @@ void tokenized_message_handler (void *cls,
   }
 }
 
+#if DELAY_FORWARDS
 struct DelayedMessageContext
 {
   struct GNUNET_PeerIdentity dest;
@@ -1249,10 +1296,12 @@ void send_message_delayed (void *cls,
                    default_dv_priority,
                    msg_ctx->uid,
                    GNUNET_TIME_relative_get_forever());
+      GNUNET_free(msg_ctx->message);
+      GNUNET_free(msg_ctx);
     }
-  GNUNET_free(msg_ctx->message);
-  GNUNET_free(msg_ctx);
 }
+#endif
+
 
 /**
  * Core handler for dv data messages.  Whatever this message
@@ -1267,10 +1316,10 @@ void send_message_delayed (void *cls,
  * @param distance the distance to the immediate peer
  */
 static int handle_dv_data_message (void *cls,
-                             const struct GNUNET_PeerIdentity * peer,
-                             const struct GNUNET_MessageHeader * message,
-                             struct GNUNET_TIME_Relative latency,
-                             uint32_t distance)
+                                   const struct GNUNET_PeerIdentity * peer,
+                                   const struct GNUNET_MessageHeader * message,
+                                   struct GNUNET_TIME_Relative latency,
+                                   uint32_t distance)
 {
   const p2p_dv_MESSAGE_Data *incoming = (const p2p_dv_MESSAGE_Data *) message;
   const struct GNUNET_MessageHeader *packed_message;
@@ -1282,21 +1331,22 @@ static int handle_dv_data_message (void *cls,
   struct GNUNET_PeerIdentity *destination;
   struct FindDestinationContext fdc;
   struct TokenizedMessageContext tkm_ctx;
+  int i;
+  int found_pos;
+#if DELAY_FORWARDS
   struct DelayedMessageContext *delayed_context;
+#endif
 #if USE_PEER_ID
   struct CheckPeerContext checkPeerCtx;
 #endif
+#if DEBUG_DV_MESSAGES
   char *sender_id;
-  char *direct_id;
+#endif
   int ret;
   size_t packed_message_size;
   char *cbuf;
-#if NO_MST
-  size_t offset;
-#endif
-  packed_message_size = ntohs(incoming->header.size) - sizeof(p2p_dv_MESSAGE_Data);
-
 
+  packed_message_size = ntohs(incoming->header.size) - sizeof(p2p_dv_MESSAGE_Data);
 #if DEBUG_DV
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "%s: Receives DATA message from %s size %d, packed size %d!\n", my_short_id, GNUNET_i2s(peer) , ntohs(incoming->header.size), packed_message_size);
@@ -1313,15 +1363,10 @@ static int handle_dv_data_message (void *cls,
     }
 
   dn = GNUNET_CONTAINER_multihashmap_get (direct_neighbors,
-                                  &peer->hashPubKey);
+                                          &peer->hashPubKey);
   if (dn == NULL)
-    {
-#if DEBUG_DV
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: dn NULL!\n", "dv");
-#endif
-      return GNUNET_OK;
-    }
+    return GNUNET_OK;
+
   sid = ntohl (incoming->sender);
 #if USE_PEER_ID
   if (sid != 0)
@@ -1344,12 +1389,9 @@ static int handle_dv_data_message (void *cls,
 
   if (pos == NULL)
     {
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
 #if DEBUG_DV_MESSAGES
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: unknown sender (%u), Message uid %llu from %s!\n", GNUNET_i2s(&my_identity), ntohl(incoming->sender), ntohl(incoming->uid), direct_id);
-#endif
-      GNUNET_free(direct_id);
+                  "%s: unknown sender (%u), Message uid %u from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
       pos = dn->referee_head;
       while ((NULL != pos) && (pos->referrer_id != sid))
       {
@@ -1358,13 +1400,34 @@ static int handle_dv_data_message (void *cls,
         GNUNET_free(sender_id);
         pos = pos->next;
       }
-
-#if DEBUG_MESSAGE_DROP
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: DROPPING MESSAGE type %d, unknown sender! Message immediately from %s!\n", GNUNET_i2s(&my_identity), ntohs(((struct GNUNET_MessageHeader *)&incoming[1])->type), direct_id);
-      GNUNET_free(direct_id);
 #endif
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "%s: unknown sender (%u), Message uid %u from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
+
+      found_pos = -1;
+      for (i = 0; i< MAX_OUTSTANDING_MESSAGES; i++)
+        {
+          if (dn->pending_messages[i].sender_id == 0)
+            {
+              found_pos = i;
+              break;
+            }
+        }
+
+      if (found_pos == -1)
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                      "%s: Too many unknown senders (%u), ignoring message! Message uid %llu from %s!\n", my_short_id, ntohl(incoming->sender), ntohl(incoming->uid), GNUNET_i2s(&dn->identity));
+        }
+      else
+        {
+            dn->pending_messages[found_pos].message = GNUNET_malloc(ntohs (message->size));
+            memcpy(dn->pending_messages[found_pos].message, message, ntohs(message->size));
+            dn->pending_messages[found_pos].distance = distance;
+            dn->pending_messages[found_pos].latency = latency;
+            memcpy(&dn->pending_messages[found_pos].sender, peer, sizeof(struct GNUNET_PeerIdentity));
+            dn->pending_messages[found_pos].sender_id = sid;
+        }
       /* unknown sender */
       return GNUNET_OK;
     }
@@ -1388,27 +1451,6 @@ static int handle_dv_data_message (void *cls,
           GNUNET_break_op(0);
           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: %s Received corrupt data, discarding!", my_short_id, "DV SERVICE");
         }
-#if NO_MST
-      offset = 0;
-      while(offset < packed_message_size)
-        {
-          packed_message = (struct GNUNET_MessageHeader *)&cbuf[offset];
-
-          GNUNET_break_op (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_GOSSIP);
-          GNUNET_break_op (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_DATA);
-          if ( (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_GOSSIP) &&
-              (ntohs (packed_message->type) != GNUNET_MESSAGE_TYPE_DV_DATA) )
-          {
-#if DEBUG_DV_MESSAGES
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                        "%s: Receives %s message(s) for me, uid %u, total size %d cost %u from %s!\n", my_short_id, "DV DATA", ntohl(incoming->uid), ntohs(packed_message->size), pos->cost, GNUNET_i2s(&pos->identity));
-#endif
-            GNUNET_assert(memcmp(peer, &pos->identity, sizeof(struct GNUNET_PeerIdentity)) != 0);
-            send_to_plugin(peer, packed_message, ntohs(packed_message->size), &pos->identity, pos->cost);
-          }
-          offset += ntohs(packed_message->size);
-        }
-#endif
       return GNUNET_OK;
     }
   else
@@ -1436,7 +1478,7 @@ static int handle_dv_data_message (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "%s: Receives %s message uid %u for someone we don't know (id %u)!\n", my_short_id, "DV DATA", ntohl(incoming->uid), tid);
 #endif
-    return GNUNET_OK;
+      return GNUNET_OK;
     }
   destination = &fdc.dest->identity;
 
@@ -1448,9 +1490,8 @@ static int handle_dv_data_message (void *cls,
 #endif
 
 #if DEBUG_DV_MESSAGES
-      direct_id = GNUNET_strdup(GNUNET_i2s(&dn->identity));
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "%s: DROPPING MESSAGE uid %u type %d, routing loop! Message immediately from %s!\n", my_short_id, ntohl(incoming->uid), ntohs(packed_message->type), direct_id);
+                  "%s: DROPPING MESSAGE uid %u type %d, routing loop! Message immediately from %s!\n", my_short_id, ntohl(incoming->uid), ntohs(packed_message->type), GNUNET_i2s(&dn->identity));
 #endif
       return GNUNET_OK;
     }
@@ -1458,18 +1499,12 @@ static int handle_dv_data_message (void *cls,
   /* At this point we have a message, and we need to forward it on to the
    * next DV hop.
    */
-  /* FIXME: Can't send message on, we have to behave.
-   * We have to tell core we have a message for the next peer, and let
-   * transport do transport selection on how to get this message to 'em */
-  /*ret = send_message (&destination,
-                      &original_sender,
-                      packed_message, DV_PRIORITY, DV_DELAY);*/
-
 #if DEBUG_DV_MESSAGES
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "%s: FORWARD %s message for %s, uid %u, size %d type %d, cost %u!\n", my_short_id, "DV DATA", GNUNET_i2s(destination), ntohl(incoming->uid), ntohs(packed_message->size), ntohs(packed_message->type), pos->cost);
 #endif
 
+#if DELAY_FORWARDS
   if (GNUNET_TIME_absolute_get_duration(pos->last_gossip).value < GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2).value)
     {
       delayed_context = GNUNET_malloc(sizeof(struct DelayedMessageContext));
@@ -1480,10 +1515,10 @@ static int handle_dv_data_message (void *cls,
       delayed_context->message_size = packed_message_size;
       delayed_context->uid = ntohl(incoming->uid);
       GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 2500), &send_message_delayed, delayed_context);
-      //GNUNET_SCHEDULER_add_now(sched, &send_message_delayed, delayed_context);
       return GNUNET_OK;
     }
   else
+#endif
     {
       ret = send_message(destination,
                          original_sender,
@@ -2308,6 +2343,7 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
   unsigned int our_id;
   char *addr1;
   char *addr2;
+  int i;
 
 #if DEBUG_DV_PEER_NUMBERS
   char *encAbout;
@@ -2424,8 +2460,22 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
       GNUNET_CONTAINER_multihashmap_put (extended_neighbors, &peer->hashPubKey,
                                  neighbor,
                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-      if (cost != DIRECT_NEIGHBOR_COST)
+      if (referrer_peer_id != 0)
+        {
+          for (i = 0; i< MAX_OUTSTANDING_MESSAGES; i++)
+            {
+              if (referrer->pending_messages[i].sender_id == referrer_peer_id) /* We have a queued message from just learned about peer! */
+                {
+#if DEBUG_DV_MESSAGES
+                  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: learned about peer %llu from which we have a previous unknown message, processing!\n", my_short_id, referrer_peer_id);
+#endif
+                  handle_dv_data_message(NULL, &referrer->pending_messages[i].sender, referrer->pending_messages[i].message, referrer->pending_messages[i].latency, referrer->pending_messages[i].distance);
+                  GNUNET_free(referrer->pending_messages[i].message);
+                  referrer->pending_messages[i].sender_id = 0;
+                }
+            }
+        }
+      if ((cost != DIRECT_NEIGHBOR_COST) && (neighbor->pkey != NULL))
         {
           /* Added neighbor, now send HELLO to transport */
           hello_context = GNUNET_malloc(sizeof(struct HelloContext));
@@ -2436,7 +2486,9 @@ addUpdateNeighbor (const struct GNUNET_PeerIdentity * peer, struct GNUNET_CRYPTO
           GNUNET_assert(memcmp(hello_context->direct_peer, &hello_context->distant_peer, sizeof(struct GNUNET_PeerIdentity)) != 0);
           addr1 = GNUNET_strdup(GNUNET_i2s(hello_context->direct_peer));
           addr2 = GNUNET_strdup(GNUNET_i2s(&hello_context->distant_peer));
+#if DEBUG_DV
           GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: GIVING HELLO size %d for %s via %s to TRANSPORT\n", my_short_id, GNUNET_HELLO_size(hello_msg), addr2, addr1);
+#endif
           GNUNET_free(addr1);
           GNUNET_free(addr2);
           send_to_plugin(hello_context->direct_peer, GNUNET_HELLO_get_header(hello_msg), GNUNET_HELLO_size(hello_msg), &hello_context->distant_peer, cost);
@@ -2501,8 +2553,10 @@ static int handle_dv_disconnect_message (void *cls,
       if (distant->referrer_id == ntohl(enc_message->peer_id))
         {
           distant_neighbor_free(distant);
+          distant = referrer->referee_head;
         }
-      distant = referrer->referee_head;
+      else
+        distant = distant->next;
     }
 
   return GNUNET_OK;
@@ -2709,12 +2763,11 @@ static int add_all_direct_neighbors (void *cls,
  * @param cls closure
  * @param peer id of the peer, NULL for last call
  * @param hello hello message for the peer (can be NULL)
- * @param trust amount of trust we have in the peer
  */
 static void
 process_peerinfo (void *cls,
                   const struct GNUNET_PeerIdentity *peer,
-                  const struct GNUNET_HELLO_Message *hello, uint32_t trust)
+                  const struct GNUNET_HELLO_Message *hello)
 {
   struct PeerIteratorContext *peerinfo_iterator = cls;
   struct DirectNeighbor *neighbor = peerinfo_iterator->neighbor;
@@ -2733,7 +2786,6 @@ process_peerinfo (void *cls,
 #endif
           peerinfo_iterator->ic = GNUNET_PEERINFO_iterate(peerinfo_handle,
                                                           &peerinfo_iterator->neighbor->identity,
-                                                          0,
                                                           GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3),
                                                           &process_peerinfo,
                                                           peerinfo_iterator);
@@ -2757,12 +2809,19 @@ process_peerinfo (void *cls,
         }
 
       sent = GNUNET_CONTAINER_multihashmap_iterate (extended_neighbors, &add_all_extended_peers, neighbor->send_context);
-
+      if (stats != NULL)
+        {
+          GNUNET_STATISTICS_update (stats, "# distant peers gossiped to direct neighbors", sent, GNUNET_NO);
+        }
 #if DEBUG_DV_PEER_NUMBERS
       neighbor_pid = GNUNET_strdup(GNUNET_i2s(&neighbor->identity));
       GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Gossipped %d extended peers to %s\n", GNUNET_i2s(&my_identity), sent, neighbor_pid);
 #endif
       sent = GNUNET_CONTAINER_multihashmap_iterate (direct_neighbors, &add_all_direct_neighbors, neighbor);
+      if (stats != NULL)
+        {
+          GNUNET_STATISTICS_update (stats, "# direct peers gossiped to direct neighbors", sent, GNUNET_NO);
+        }
 #if DEBUG_DV_PEER_NUMBERS
       GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "%s: Gossipped about %s to %d direct peers\n", GNUNET_i2s(&my_identity), neighbor_pid, sent);
       GNUNET_free(neighbor_pid);
@@ -2810,7 +2869,6 @@ void handle_core_connect (void *cls,
     peerinfo_iterator->neighbor = neighbor;
     peerinfo_iterator->ic = GNUNET_PEERINFO_iterate (peerinfo_handle,
                                                      peer,
-                                                     0,
                                                      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3),
                                                      &process_peerinfo,
                                                      peerinfo_iterator);
@@ -2830,7 +2888,13 @@ void handle_core_connect (void *cls,
   {
     about = GNUNET_CONTAINER_multihashmap_get(extended_neighbors, &peer->hashPubKey);
     if ((GNUNET_CONTAINER_multihashmap_get(direct_neighbors, &peer->hashPubKey) == NULL) && (about != NULL))
-      sent = GNUNET_CONTAINER_multihashmap_iterate(direct_neighbors, &add_distant_all_direct_neighbors, about);
+      {
+        sent = GNUNET_CONTAINER_multihashmap_iterate(direct_neighbors, &add_distant_all_direct_neighbors, about);
+        if (stats != NULL)
+          {
+            GNUNET_STATISTICS_update (stats, "# direct peers gossiped to new direct neighbors", sent, GNUNET_NO);
+          }
+      }
 #if DEBUG_DV
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "%s: Distance (%d) greater than %d or already know about peer (%s), not re-adding!\n", "dv", distance, DIRECT_NEIGHBOR_COST, GNUNET_i2s(peer));
@@ -2942,6 +3006,7 @@ run (void *cls,
                        &core_init,
                        &handle_core_connect,
                        &handle_core_disconnect,
+                      NULL,
                        NULL,
                        GNUNET_NO,
                        NULL,
@@ -2951,8 +3016,7 @@ run (void *cls,
   if (coreAPI == NULL)
     return;
 
-  coreMST = GNUNET_SERVER_mst_create (GNUNET_SERVER_MAX_MESSAGE_SIZE,
-                                      &tokenized_message_handler,
+  coreMST = GNUNET_SERVER_mst_create (&tokenized_message_handler,
                                       NULL);
 
    peerinfo_handle = GNUNET_PEERINFO_connect(sched, cfg);
@@ -2968,12 +3032,6 @@ run (void *cls,
                                 GNUNET_TIME_UNIT_FOREVER_REL,
                                 &shutdown_task,
                                 NULL);
-#if INSANE_GOSSIP
-  GNUNET_SCHEDULER_add_delayed (sched,
-                                GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 5),
-                                &gossip_all_to_all,
-                                NULL);
-#endif
 }