fs hackery

diff --git a/TODO b/TODO
index 01b3d5a95b93d0ded45bd8565301c4aad9d7383d..f906e671383a0160463fd5f4680b2f85433d567e 100644 (file)
--- a/TODO
+++ b/TODO
@@ -4,10 +4,11 @@
     (disconnect does not really succeed, or what?)
     Also, core does not seem to actually decrement the number of active
     sessions even if no connections exist for a while! (likely related...)
-  - transport reports bw quota violations on slow systems (core or
-    transport issue? tracking issue or actual violation?)
 * FS: [CG]
   - 2-peer download is still too slow (why? testcases fail on slow systems!)
+    => now seems to work well with high BW quota ("\infty"), but not so well with the 
+       typical small quota
+    => should print kb/s in testcase output!
 * WWW:
   - Get IPv6 hooked up [AK, after April 12th]
   - change DNS [CG, need DNS]

diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf
index 83d995adcbf5d9cf732c38cb2cd8decb41c701fd..820c1e6391554f6603f16f3fbd54dc16da3f0517 100644 (file)
--- a/src/fs/fs_test_lib_data.conf
+++ b/src/fs/fs_test_lib_data.conf
@@ -41,8 +41,8 @@ HOSTNAME = localhost
 [core]
 PORT = 43470
 HOSTNAME = localhost
-#TOTAL_QUOTA_IN = 3932160
-#TOTAL_QUOTA_OUT = 3932160
+TOTAL_QUOTA_IN = 3932160
+TOTAL_QUOTA_OUT = 3932160
 #DEBUG = YES
 #PREFIX = valgrind --tool=memcheck --leak-check=yes
 #BINARY = /home/grothoff/bin/gnunet-service-core
@@ -50,6 +50,7 @@ HOSTNAME = localhost
 [fs]
 PORT = 43471
 HOSTNAME = localhost
+#OPTIONS = -L DEBUG
 #DEBUG = YES
 #PREFIX = valgrind --tool=memcheck --leak-check=yes
 #BINARY = /home/grothoff/bin/gnunet-service-fs

diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index 13f1c727cc88a3dc77e2051d9b0e3b8bef51ef93..95d3263debc580a1e60f871fe6d3eac3ea74d6fa 100644 (file)
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -47,21 +47,22 @@
 #include "gnunet-service-fs_indexing.h"
 #include "fs.h"
 
-#define DEBUG_FS GNUNET_NO
+#define DEBUG_FS GNUNET_YES
 
 /**
  * Maximum number of outgoing messages we queue per peer.
- * FIXME: set to a tiny value for testing; make configurable.
+ * FIXME: make configurable?
  */
-#define MAX_QUEUE_PER_PEER 2
+#define MAX_QUEUE_PER_PEER 16
 
 /**
  * Inverse of the probability that we will submit the same query
  * to the same peer again.  If the same peer already got the query
  * repeatedly recently, the probability is multiplied by the inverse
- * of this number each time.
+ * of this number each time.  Note that we only try about every TTL_DECREMENT/2
+ * plus MAX_CORK_DELAY (so roughly every 3.5s).
  */
-#define RETRY_PROBABILITY_INV 8
+#define RETRY_PROBABILITY_INV 3
 
 /**
  * What is the maximum delay for a P2P FS message (in our interaction
@@ -77,9 +78,9 @@
 /**
  * Maximum number of requests (from other peers) that we're
  * willing to have pending at any given point in time.
- * FIXME: set from configuration (and 32 is a tiny value for testing only).
+ * FIXME: set from configuration.
  */
-static uint64_t max_pending_requests = 32;
+static uint64_t max_pending_requests = (32 * 1024);
 
 
 /**
@@ -541,6 +542,11 @@ struct PendingRequest
    */
   uint32_t type;
 
+  /**
+   * Remove this request after transmission of the current response.
+   */
+  int do_remove;
+
 };
 
 
@@ -891,7 +897,12 @@ handle_client_disconnect (void *cls,
   if (pos == NULL)
     return; /* no requests pending for this client */
   while (NULL != (rcl = pos->rl_head))
-    destroy_pending_request (rcl->req);
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                 "Destroying pending request `%s' on disconnect\n",
+                 GNUNET_h2s (&rcl->req->query));
+      destroy_pending_request (rcl->req);
+    }
   if (prev == NULL)
     client_list = pos->next;
   else
@@ -999,7 +1010,8 @@ transmit_to_peer (void *cls,
   struct GNUNET_PeerIdentity pid;
   struct PendingMessage *pm;
   size_t msize;
-  
+  struct PendingRequest *pr;
+ 
   cp->cth = NULL;
   if (NULL == buf)
     {
@@ -1016,6 +1028,7 @@ transmit_to_peer (void *cls,
       memcpy (&cbuf[msize], &pm[1], pm->msize);
       msize += pm->msize;
       size -= pm->msize;
+      pr = pm->pml->req;
       destroy_pending_message (pm, cp->pid);
     }
   if (NULL != pm)
@@ -1482,13 +1495,31 @@ target_peer_select_cb (void *cls,
   struct PendingRequest *pr = psc->pr;
   double score;
   unsigned int i;
+  unsigned int pc;
   
   /* 1) check if we have already (recently) forwarded to this peer */
+  pc = 0;
   for (i=0;i<pr->used_pids_off;i++)
-    if ( (pr->used_pids[i] == cp->pid) &&
-        (0 != GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                        RETRY_PROBABILITY_INV)) )
-      return GNUNET_YES; /* skip */
+    if (pr->used_pids[i] == cp->pid) 
+      {
+       pc++;
+       if (0 != GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                          RETRY_PROBABILITY_INV))
+         {
+#if DEBUG_FS
+           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                       "NOT re-trying query that was previously transmitted %u times\n",
+                       (unsigned int) pr->used_pids_off);
+#endif
+           return GNUNET_YES; /* skip */
+         }
+      }
+#if DEBUG_FS
+  if (0 < pc)
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               "Re-trying query that was previously transmitted %u times to this peer\n",
+               (unsigned int) pc);
+#endif
   // 2) calculate how much we'd like to forward to this peer
   score = 42; // FIXME!
   // FIXME: also need API to gather data on responsiveness
@@ -1524,10 +1555,18 @@ forward_request_task (void *cls,
   struct PendingRequest *pr = cls;
   struct PeerSelectionContext psc;
   struct ConnectedPeer *cp; 
+  struct GNUNET_TIME_Relative delay;
 
   pr->task = GNUNET_SCHEDULER_NO_TASK;
   if (pr->irc != NULL)
-    return; /* already pending */
+    {
+#if DEBUG_FS
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Forwarding of query `%s' not attempted due to pending local lookup!\n",
+                 GNUNET_h2s (&pr->query));
+#endif
+      return; /* already pending */
+    }
   /* (1) select target */
   psc.pr = pr;
   psc.target_score = DBL_MIN;
@@ -1536,13 +1575,15 @@ forward_request_task (void *cls,
                                         &psc);  
   if (psc.target_score == DBL_MIN)
     {
-#if DEBUG_FS
+      delay = get_processing_delay ();
+#if DEBUG_FS 
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "No peer selected for forwarding of query `%s'!\n",
-                 GNUNET_h2s (&pr->query));
+                 "No peer selected for forwarding of query `%s', will try again in %llu ms!\n",
+                 GNUNET_h2s (&pr->query),
+                 delay.value);
 #endif
       pr->task = GNUNET_SCHEDULER_add_delayed (sched,
-                                              get_processing_delay (),
+                                              delay,
                                               &forward_request_task,
                                               pr);
       return; /* nobody selected */
@@ -1864,7 +1905,6 @@ process_reply (void *cls,
   GNUNET_HashCode chash;
   GNUNET_HashCode mhash;
   size_t msize;
-  int do_remove;
 
 #if DEBUG_FS
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1876,7 +1916,6 @@ process_reply (void *cls,
                            gettext_noop ("# replies received and matched"),
                            1,
                            GNUNET_NO);
-  do_remove = GNUNET_NO;
   GNUNET_CRYPTO_hash (prq->data,
                      prq->size,
                      &chash);
@@ -1895,7 +1934,17 @@ process_reply (void *cls,
          GNUNET_FS_drq_get_cancel (pr->drq);
          pr->drq = NULL;
        }
-      do_remove = GNUNET_YES;
+      pr->do_remove = GNUNET_YES;
+      if (pr->task != GNUNET_SCHEDULER_NO_TASK)
+       {
+         GNUNET_SCHEDULER_cancel (sched,
+                                  pr->task);
+         pr->task = GNUNET_SCHEDULER_NO_TASK;
+       }
+      GNUNET_break (GNUNET_YES ==
+                   GNUNET_CONTAINER_multihashmap_remove (query_request_map,
+                                                         key,
+                                                         pr));
       break;
     case GNUNET_DATASTORE_BLOCKTYPE_SBLOCK:
       if (pr->namespace == NULL)
@@ -2002,6 +2051,8 @@ process_reply (void *cls,
                                                        cl);
        }
       GNUNET_break (cl->th != NULL);
+      if (pr->do_remove)               
+       destroy_pending_request (pr);           
     }
   else
     {
@@ -2030,20 +2081,6 @@ process_reply (void *cls,
       memcpy (&pm[1], prq->data, prq->size);
       add_to_pending_messages_for_peer (cp, reply, pr);
     }
-  if (GNUNET_YES == do_remove)
-    {
-#if DEBUG_FS
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Removing request `%s' from request map (has been satisfied)\n",
-                 GNUNET_h2s (key));
-#endif
-      GNUNET_break (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (query_request_map,
-                                                         key,
-                                                         pr));
-      // FIXME: request somehow does not fully disappear; how to fix? 
-      // destroy_pending_request (pr); (not like this!)
-    }
   // FIXME: implement hot-path routing statistics keeping!
   return GNUNET_YES;
 }
@@ -2373,21 +2410,25 @@ process_local_reply (void *cls,
   prq.type = type;
   prq.priority = priority;  
   process_reply (&prq, key, pr);
-  
-  if ( ( (pr->client_request_list == NULL) &&
-        ( (GNUNET_YES == test_load_too_high()) ||
-          (pr->results_found > 5 + 2 * pr->priority) ) ) ||
-       (type == GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) ) 
+
+  if ( (type == GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) ||
+       (type == GNUNET_DATASTORE_BLOCKTYPE_IBLOCK) ) 
+    {
+      GNUNET_FS_drq_get_next (GNUNET_NO);
+      return;
+    }
+  if ( (pr->client_request_list == NULL) &&
+       ( (GNUNET_YES == test_load_too_high()) ||
+        (pr->results_found > 5 + 2 * pr->priority) ) )
     {
 #if DEBUG_FS > 2
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Unique reply found or load too high, done with request\n");
+                 "Load too high, done with request\n");
 #endif
-      if (type != GNUNET_DATASTORE_BLOCKTYPE_DBLOCK)
-       GNUNET_STATISTICS_update (stats,
-                                 gettext_noop ("# processing result set cut short due to load"),
-                                 1,
-                                 GNUNET_NO);
+      GNUNET_STATISTICS_update (stats,
+                               gettext_noop ("# processing result set cut short due to load"),
+                               1,
+                               GNUNET_NO);
       GNUNET_FS_drq_get_next (GNUNET_NO);
       return;
     }
@@ -2591,7 +2632,7 @@ handle_p2p_get (void *cls,
       return GNUNET_OK;
     }
 
-#if DEBUG_FS
+#if DEBUG_FS 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Received request for `%s' of type %u from peer `%4s' with flags %u\n",
              GNUNET_h2s (&gm->query),

diff --git a/src/fs/gnunet-service-fs_drq.c b/src/fs/gnunet-service-fs_drq.c
index dea29df7ecee351d886310c6a03c4617910d3800..43e138e131439c0359e1ac0a36ef7f16b3159dd4 100644 (file)
--- a/src/fs/gnunet-service-fs_drq.c
+++ b/src/fs/gnunet-service-fs_drq.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet-service-fs_drq.h"
 
-#define DEBUG_DRQ GNUNET_YES
+#define DEBUG_DRQ GNUNET_NO
 
 /**
  * Signature of a function that is called whenever a datastore
@@ -219,7 +219,9 @@ run_next_request (void *cls,
              GNUNET_h2s (&gc->key),
              gc->type);
 #endif
-  GNUNET_DATASTORE_get (dsh, &gc->key, gc->type, 
+  GNUNET_DATASTORE_get (dsh, 
+                       &gc->key,
+                       gc->type, 
                        &get_iterator,
                        gc,
                        GNUNET_TIME_absolute_get_remaining(gc->timeout));
@@ -306,6 +308,16 @@ shutdown_task (void *cls,
       GNUNET_free (drq);
     }
   drq_tail = NULL;
+  if (drq_running != NULL)
+    {
+      GNUNET_SCHEDULER_cancel (sched,
+                              drq_running->task);
+      drq_running->iter (drq_running->iter_cls,
+                        NULL, 0, NULL, 0, 0, 0, 
+                        GNUNET_TIME_UNIT_ZERO_ABS, 0);
+      GNUNET_free (drq_running);
+      drq_running = NULL;
+    }
 }
 
 

diff --git a/src/fs/test_gnunet_service_fs_p2p.c b/src/fs/test_gnunet_service_fs_p2p.c
index 4153c65848b7192d966e2ec06cda1be4ead34fbf..024e08155a6afb3fa4c3034c341d47d03d113824 100644 (file)
--- a/src/fs/test_gnunet_service_fs_p2p.c
+++ b/src/fs/test_gnunet_service_fs_p2p.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "fs_test_lib.h"
 
-#define VERBOSE GNUNET_NO
+#define VERBOSE GNUNET_YES
 
 /**
  * File-size we use for testing.