avoid tracking requests twice if multiple blocks in the same file are identical
authorChristian Grothoff <christian@grothoff.org>
Mon, 1 Mar 2010 16:50:43 +0000 (16:50 +0000)
committerChristian Grothoff <christian@grothoff.org>
Mon, 1 Mar 2010 16:50:43 +0000 (16:50 +0000)
src/fs/fs_download.c
src/fs/gnunet-service-fs.c

index aa78a929450d8301d492fd748e7271cd19fe0ed2..f873f80de9e57ccb03dbc1f9b552edb7d9b0ffb0 100644 (file)
@@ -303,50 +303,69 @@ calculate_block_size (uint64_t fsize,
 
 
 /**
- * Process a download result.
- *
- * @param dc our download context
- * @param type type of the result
- * @param data the (encrypted) response
- * @param size size of data
+ * Closure for iterator processing results.
  */
-static void
-process_result (struct GNUNET_FS_DownloadContext *dc,
-               uint32_t type,
-               const void *data,
-               size_t size)
+struct ProcessResultClosure
 {
-  struct GNUNET_FS_ProgressInfo pi;
+  
+  /**
+   * Hash of data.
+   */
   GNUNET_HashCode query;
-  struct DownloadRequest *sm;
+
+  /**
+   * Data found in P2P network.
+   */ 
+  const void *data;
+
+  /**
+   * Our download context.
+   */
+  struct GNUNET_FS_DownloadContext *dc;
+               
+  /**
+   * Number of bytes in data.
+   */
+  size_t size;
+
+  /**
+   * Type of data.
+   */
+  uint32_t type;
+  
+};
+
+
+/**
+ * Iterator over entries in the pending requests in the 'active' map for the
+ * reply that we just got.
+ *
+ * @param cls closure (our 'struct ProcessResultClosure')
+ * @param value value in the hash map (a 'struct DownloadRequest')
+ * @return GNUNET_YES (we should continue to iterate); unless serious error
+ */
+static int
+process_result_with_request (void *cls,
+                            const GNUNET_HashCode * key,
+                            void *value)
+{
+  struct ProcessResultClosure *prc = cls;
+  struct DownloadRequest *sm = value;
+  struct GNUNET_FS_DownloadContext *dc = prc->dc;
   struct GNUNET_CRYPTO_AesSessionKey skey;
   struct GNUNET_CRYPTO_AesInitializationVector iv;
-  char pt[size];
+  char pt[prc->size];
+  struct GNUNET_FS_ProgressInfo pi;
   uint64_t off;
   size_t app;
   int i;
   struct ContentHashKey *chk;
   char *emsg;
 
-
-  GNUNET_CRYPTO_hash (data, size, &query);
-#if DEBUG_DOWNLOAD
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Received result for query `%s' from `%s'-service\n",
-             GNUNET_h2s (&query),
-             "FS");
-#endif
-  sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
-                                         &query);
-  if (NULL == sm)
-    {
-      GNUNET_break (0);
-      return;
-    }
-  if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
-                                   dc->treedepth,
-                                   sm->offset,
-                                   sm->depth))
+  if (prc->size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
+                                        dc->treedepth,
+                                        sm->offset,
+                                        sm->depth))
     {
 #if DEBUG_DOWNLOAD
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -355,7 +374,7 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
                                        dc->treedepth,
                                        sm->offset,
                                        sm->depth),
-                 size);
+                 prc->size);
 #endif
       dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
       /* signal error */
@@ -372,15 +391,15 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
        }
       GNUNET_CLIENT_disconnect (dc->client);
       dc->client = NULL;
-      return;
+      return GNUNET_NO;
     }
   GNUNET_assert (GNUNET_YES ==
                 GNUNET_CONTAINER_multihashmap_remove (dc->active,
-                                                      &query,
+                                                      &prc->query,
                                                       sm));
   GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
-  GNUNET_CRYPTO_aes_decrypt (data,
-                            size,
+  GNUNET_CRYPTO_aes_decrypt (prc->data,
+                            prc->size,
                             &skey,
                             &iv,
                             pt);
@@ -408,13 +427,13 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
                         (unsigned long long) off,
                         dc->filename,
                         STRERROR (errno));
-      else if (size !=
+      else if (prc->size !=
               GNUNET_DISK_file_write (dc->handle,
                                       pt,
-                                      size))
+                                      prc->size))
        GNUNET_asprintf (&emsg,
                         _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
-                        (unsigned int) size,
+                        (unsigned int) prc->size,
                         (unsigned long long) off,
                         dc->filename,
                         STRERROR (errno));
@@ -439,12 +458,12 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
          GNUNET_CLIENT_disconnect (dc->client);
          dc->client = NULL;
          GNUNET_free (sm);
-         return;
+         return GNUNET_NO;
        }
     }
   if (sm->depth == dc->treedepth) 
     {
-      app = size;
+      app = prc->size;
       if (sm->offset < dc->offset)
        {
          /* starting offset begins in the middle of pt,
@@ -452,12 +471,12 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
          GNUNET_assert (app > (dc->offset - sm->offset));
          app -= (dc->offset - sm->offset);       
        }
-      if (sm->offset + size > dc->offset + dc->length)
+      if (sm->offset + prc->size > dc->offset + dc->length)
        {
          /* end of block is after relevant range,
             do not count last bytes as progress */
-         GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
-         app -= (sm->offset + size) - (dc->offset + dc->length);
+         GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length));
+         app -= (sm->offset + prc->size) - (dc->offset + dc->length);
        }
       dc->completed += app;
     }
@@ -466,7 +485,7 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
   make_download_status (&pi, dc);
   pi.value.download.specifics.progress.data = pt;
   pi.value.download.specifics.progress.offset = sm->offset;
-  pi.value.download.specifics.progress.data_len = size;
+  pi.value.download.specifics.progress.data_len = prc->size;
   pi.value.download.specifics.progress.depth = sm->depth;
   dc->client_info = dc->h->upcb (dc->h->upcb_cls,
                                 &pi);
@@ -500,7 +519,7 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
   if (sm->depth == dc->treedepth) 
     {
       GNUNET_free (sm);      
-      return;
+      return GNUNET_YES;
     }
 #if DEBUG_DOWNLOAD
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -508,9 +527,9 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
              sm->depth,
              (unsigned long long) sm->offset);
 #endif
-  GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
+  GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
   chk = (struct ContentHashKey*) pt;
-  for (i=(size / sizeof(struct ContentHashKey))-1;i>=0;i--)
+  for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
     {
       off = compute_dblock_offset (sm->offset,
                                   sm->depth,
@@ -524,6 +543,41 @@ process_result (struct GNUNET_FS_DownloadContext *dc,
                                 sm->depth + 1);
     }
   GNUNET_free (sm);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Process a download result.
+ *
+ * @param dc our download context
+ * @param type type of the result
+ * @param data the (encrypted) response
+ * @param size size of data
+ */
+static void
+process_result (struct GNUNET_FS_DownloadContext *dc,
+               uint32_t type,
+               const void *data,
+               size_t size)
+{
+  struct ProcessResultClosure prc;
+
+  prc.dc = dc;
+  prc.data = data;
+  prc.size = size;
+  prc.type = type;
+  GNUNET_CRYPTO_hash (data, size, &prc.query);
+#if DEBUG_DOWNLOAD
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Received result for query `%s' from `%s'-service\n",
+             GNUNET_h2s (&prc.query),
+             "FS");
+#endif
+  GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
+                                             &prc.query,
+                                             &process_result_with_request,
+                                             &prc);
 }
 
 
index 0a49f9a37773fea52882f8ce575fea58e83c6c8a..06d949e2e72e3961545493d60058a343df9b18e8 100644 (file)
@@ -658,9 +658,9 @@ destroy_pending_request (struct PendingRequest *pr)
     {
       GNUNET_PEER_resolve (pr->cp->pid,
                           &pid);
-      GNUNET_CONTAINER_multihashmap_remove (peer_request_map,
-                                           &pid.hashPubKey,
-                                           pr);
+      (void) GNUNET_CONTAINER_multihashmap_remove (peer_request_map,
+                                                  &pid.hashPubKey,
+                                                  pr);
       pr->cp = NULL;
     }
   if (pr->bf != NULL)
@@ -1924,6 +1924,56 @@ handle_p2p_put (void *cls,
 /* **************************** P2P GET Handling ************************ */
 
 
+/**
+ * Closure for 'check_duplicate_request_{peer,client}'.
+ */
+struct CheckDuplicateRequestClosure
+{
+  /**
+   * The new request we should check if it already exists.
+   */
+  const struct PendingRequest *pr;
+
+  /**
+   * Existing request found by the checker, NULL if none.
+   */
+  struct PendingRequest *have;
+};
+
+
+/**
+ * Iterator over entries in the 'query_request_map' that
+ * tries to see if we have the same request pending from
+ * the same client already.
+ *
+ * @param cls closure (our 'struct CheckDuplicateRequestClosure')
+ * @param key current key code (query, ignored, must match)
+ * @param value value in the hash map (a 'struct PendingRequest' 
+ *              that already exists)
+ * @return GNUNET_YES if we should continue to
+ *         iterate (no match yet)
+ *         GNUNET_NO if not (match found).
+ */
+static int
+check_duplicate_request_client (void *cls,
+                               const GNUNET_HashCode * key,
+                               void *value)
+{
+  struct CheckDuplicateRequestClosure *cdc = cls;
+  struct PendingRequest *have = value;
+
+  if (have->client_request_list == NULL)
+    return GNUNET_YES;
+  if ( (cdc->pr->client_request_list->client_list->client == have->client_request_list->client_list->client) &&
+       (cdc->pr != have) )
+    {
+      cdc->have = have;
+      return GNUNET_NO;
+    }
+  return GNUNET_YES;
+}
+
+
 /**
  * We're processing (local) results for a search request
  * from another peer.  Pass applicable results to the
@@ -1955,6 +2005,7 @@ process_local_reply (void *cls,
 {
   struct PendingRequest *pr = cls;
   struct ProcessReplyClosure prq;
+  struct CheckDuplicateRequestClosure cdrc;
   GNUNET_HashCode dhash;
   GNUNET_HashCode mhash;
   GNUNET_HashCode query;
@@ -1967,8 +2018,30 @@ process_local_reply (void *cls,
 #endif
       pr->drq = NULL;
       if (pr->client_request_list != NULL)
-       GNUNET_SERVER_receive_done (pr->client_request_list->client_list->client, 
-                                   GNUNET_YES);
+       {
+         GNUNET_SERVER_receive_done (pr->client_request_list->client_list->client, 
+                                     GNUNET_YES);
+         /* Figure out if this is a duplicate request and possibly
+            merge 'struct PendingRequest' entries */
+         cdrc.have = NULL;
+         cdrc.pr = pr;
+         GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map,
+                                                     &pr->query,
+                                                     &check_duplicate_request_client,
+                                                     &cdrc);
+         if (cdrc.have != NULL)
+           {
+#if DEBUG_FS
+             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                         "Received request for block `%s' twice from client, will only request once.\n",
+                         GNUNET_h2s (&pr->query));
+#endif
+             
+             destroy_pending_request (pr);
+             return;
+           }
+       }
+
       /* no more results */
       if (pr->task == GNUNET_SCHEDULER_NO_TASK)
        pr->task = GNUNET_SCHEDULER_add_now (sched,
@@ -2105,23 +2178,6 @@ bound_priority (uint32_t prio_in,
 }
 
 
-/**
- * Closure for 'check_duplicate_request'.
- */
-struct CheckDuplicateRequestClosure
-{
-  /**
-   * The new request we should check if it already exists.
-   */
-  const struct PendingRequest *pr;
-
-  /**
-   * Existing request found by the checker, NULL if none.
-   */
-  struct PendingRequest *have;
-};
-
-
 /**
  * Iterator over entries in the 'query_request_map' that
  * tries to see if we have the same request pending from
@@ -2136,9 +2192,9 @@ struct CheckDuplicateRequestClosure
  *         GNUNET_NO if not (match found).
  */
 static int
-check_duplicate_request (void *cls,
-                        const GNUNET_HashCode * key,
-                        void *value)
+check_duplicate_request_peer (void *cls,
+                             const GNUNET_HashCode * key,
+                             void *value)
 {
   struct CheckDuplicateRequestClosure *cdc = cls;
   struct PendingRequest *have = value;
@@ -2327,7 +2383,7 @@ handle_p2p_get (void *cls,
   cdc.pr = pr;
   GNUNET_CONTAINER_multihashmap_get_multiple (query_request_map,
                                              &gm->query,
-                                             &check_duplicate_request,
+                                             &check_duplicate_request_peer,
                                              &cdc);
   if (cdc.have != NULL)
     {