fully implement in-line files in directories

diff --git a/TODO b/TODO
index b5d21b828d2ffd9ed60e111e95fe44b507dc5779..a4393324c747d020281d7fcd056364d5e1152e37 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,13 +1,5 @@
 0.9.0pre1:
-* MIGRATION [CG]
-  - on-demand encoding => move logic to block-library!?
-  - peer selection => how to consider latency/bw/etc?
-  - content transmission => how often the same block?
-  - how to select delay before next migration? 
-  - testing
 * FS: [CG]
-  - migration (inbound)
-  - support for in-line files in directories (FIXME in fs_download)
   - bound parallelism (# fs downloads)
   - distinguish in performance tracking and event signalling between
     downloads that are actually running and those that are merely in the queue
@@ -23,6 +15,15 @@
   - [gnunet-service-fs.c:700]: member 'ConnectedPeer::last_client_replies_woff' is never used
   - GAP improvements:
     + active reply route caching design & implementation of service; gap extension!
+* MIGRATION [CG]
+  - on-demand encoding => move logic to block-library!?
+  - peer selection => how to consider latency/bw/etc?
+  - content transmission => how often the same block?
+  - how to select delay before next migration? 
+  - migration to us
+  - testing
+  - integrate with FS or not? (peer list, index/on-demand encoding, block code,
+    inbound priority assignment; all would be easier with tight integration!)
 * TBENCH: [MW]
   - good to have for transport/DV evaluation! 
 * DV: [Nate]

diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index b179c70dc801e7823bc21edd12682f8128c9ac82..8194ad9f59ce5fcc0bf9cb5470c119c98acf8e6d 100644 (file)
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -515,6 +515,125 @@ static void
 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 
 
+/**
+ * We found an entry in a directory.  Check if the respective child
+ * already exists and if not create the respective child download.
+ *
+ * @param cls the parent download
+ * @param filename name of the file in the directory
+ * @param uri URI of the file (CHK or LOC)
+ * @param meta meta data of the file
+ * @param length number of bytes in data
+ * @param data contents of the file (or NULL if they were not inlined)
+ */
+static void 
+trigger_recursive_download (void *cls,
+                           const char *filename,
+                           const struct GNUNET_FS_Uri *uri,
+                           const struct GNUNET_CONTAINER_MetaData *meta,
+                           size_t length,
+                           const void *data);
+
+
+/**
+ * We're done downloading a directory.  Open the file and
+ * trigger all of the (remaining) child downloads.
+ *
+ * @param dc context of download that just completed
+ */
+static void
+full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
+{
+  size_t size;
+  uint64_t size64;
+  void *data;
+  struct GNUNET_DISK_FileHandle *h;
+  struct GNUNET_DISK_MapHandle *m;
+  
+  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
+  size = (size_t) size64;
+  if (size64 != (uint64_t) size)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
+      return;
+    }
+  if (dc->filename != NULL)
+    {
+      h = GNUNET_DISK_file_open (dc->filename,
+                                GNUNET_DISK_OPEN_READ,
+                                GNUNET_DISK_PERM_NONE);
+    }
+  else
+    {
+      GNUNET_assert (dc->temp_filename != NULL);
+      h = GNUNET_DISK_file_open (dc->temp_filename,
+                                GNUNET_DISK_OPEN_READ,
+                                GNUNET_DISK_PERM_NONE);
+    }
+  if (h == NULL)
+    return; /* oops */
+  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
+  if (data == NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("Directory too large for system address space\n"));
+    }
+  else
+    {
+      GNUNET_FS_directory_list_contents (size,
+                                        data,
+                                        0,
+                                        &trigger_recursive_download,
+                                        dc);         
+      GNUNET_DISK_file_unmap (m);
+    }
+  GNUNET_DISK_file_close (h);
+  if (dc->filename == NULL)
+    {
+      if (0 != UNLINK (dc->temp_filename))
+       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                 "unlink",
+                                 dc->temp_filename);
+      GNUNET_free (dc->temp_filename);
+      dc->temp_filename = NULL;
+    }
+}
+
+
+/**
+ * Check if all child-downloads have completed and
+ * if so, signal completion (and possibly recurse to
+ * parent).
+ */
+static void
+check_completed (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  struct GNUNET_FS_DownloadContext *pos;
+
+  pos = dc->child_head;
+  while (pos != NULL)
+    {
+      if ( (pos->emsg == NULL) &&
+          (pos->completed < pos->length) )
+       return; /* not done yet */
+      if ( (pos->child_head != NULL) &&
+          (pos->has_finished != GNUNET_YES) )
+       return; /* not transitively done yet */
+      pos = pos->next;
+    }
+  dc->has_finished = GNUNET_YES;
+  /* signal completion */
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
+  make_download_status (&pi, dc);
+  dc->client_info = dc->h->upcb (dc->h->upcb_cls,
+                                &pi);
+  if (dc->parent != NULL)
+    check_completed (dc->parent);  
+}
+
+
 /**
  * We found an entry in a directory.  Check if the respective child
  * already exists and if not create the respective child download.
@@ -537,6 +656,8 @@ trigger_recursive_download (void *cls,
   struct GNUNET_FS_DownloadContext *dc = cls;  
   struct GNUNET_FS_DownloadContext *cpos;
   struct GNUNET_DISK_FileHandle *fh;
+  char *temp_name;
+  const char *real_name;
   char *fn;
   char *us;
   char *ext;
@@ -632,45 +753,48 @@ trigger_recursive_download (void *cls,
       GNUNET_free_non_null (fn);
       return;
     }
-    
-  if (data != NULL) 
+
+  temp_name = NULL;
+  if ( (data != NULL) &&
+       (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
     {
-      if (full_name != NULL)
+      if (full_name == NULL)
        {
-         fh = GNUNET_DISK_file_open (full_name,
-                                     GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
-                                     GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
-         if (fh == NULL)
-           {
-             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
-                                       "open",
-                                       full_name);           
-             GNUNET_free (full_name);
-             GNUNET_free_non_null (fn);
-             return;
-           }
-         if (length != 
-             GNUNET_DISK_file_write (fh,
-                                     data,
-                                     length))
-           {
-             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
-                                       "write",
-                                       full_name);           
-           }
-         GNUNET_DISK_file_close (fh);
+         temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
+         real_name = temp_name;
        }
       else
        {
-         /* FIXME: generate 'progress' events and move to
-            instant completion! */
-         GNUNET_break (0); // FIXME: not implemented
+         real_name = full_name;
        }
+      /* write to disk, then trigger normal download which will instantly progress to completion */
+      fh = GNUNET_DISK_file_open (real_name,
+                                 GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
+                                 GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
+      if (fh == NULL)
+       {
+         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                   "open",
+                                   real_name);       
+         GNUNET_free (full_name);
+         GNUNET_free_non_null (fn);
+         return;
+       }
+      if (length != 
+         GNUNET_DISK_file_write (fh,
+                                 data,
+                                 length))
+       {
+         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                   "write",
+                                   full_name);       
+       }
+      GNUNET_DISK_file_close (fh);
     }
   GNUNET_FS_download_start (dc->h,
                            uri,
                            meta,
-                           full_name,
+                           full_name, temp_name,
                            0,
                            GNUNET_FS_uri_chk_get_file_size (uri),
                            dc->anonymity,
@@ -678,109 +802,11 @@ trigger_recursive_download (void *cls,
                            NULL,
                            dc);
   GNUNET_free_non_null (full_name);
+  GNUNET_free_non_null (temp_name);
   GNUNET_free_non_null (fn);
 }
 
 
-/**
- * We're done downloading a directory.  Open the file and
- * trigger all of the (remaining) child downloads.
- *
- * @param dc context of download that just completed
- */
-static void
-full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
-{
-  size_t size;
-  uint64_t size64;
-  void *data;
-  struct GNUNET_DISK_FileHandle *h;
-  struct GNUNET_DISK_MapHandle *m;
-  
-  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
-  size = (size_t) size64;
-  if (size64 != (uint64_t) size)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
-      return;
-    }
-  if (dc->filename != NULL)
-    {
-      h = GNUNET_DISK_file_open (dc->filename,
-                                GNUNET_DISK_OPEN_READ,
-                                GNUNET_DISK_PERM_NONE);
-    }
-  else
-    {
-      GNUNET_assert (dc->temp_filename != NULL);
-      h = GNUNET_DISK_file_open (dc->temp_filename,
-                                GNUNET_DISK_OPEN_READ,
-                                GNUNET_DISK_PERM_NONE);
-    }
-  if (h == NULL)
-    return; /* oops */
-  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
-  if (data == NULL)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("Directory too large for system address space\n"));
-    }
-  else
-    {
-      GNUNET_FS_directory_list_contents (size,
-                                        data,
-                                        0,
-                                        &trigger_recursive_download,
-                                        dc);         
-      GNUNET_DISK_file_unmap (m);
-    }
-  GNUNET_DISK_file_close (h);
-  if (dc->filename == NULL)
-    {
-      if (0 != UNLINK (dc->temp_filename))
-       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
-                                 "unlink",
-                                 dc->temp_filename);
-      GNUNET_free (dc->temp_filename);
-      dc->temp_filename = NULL;
-    }
-}
-
-
-/**
- * Check if all child-downloads have completed and
- * if so, signal completion (and possibly recurse to
- * parent).
- */
-static void
-check_completed (struct GNUNET_FS_DownloadContext *dc)
-{
-  struct GNUNET_FS_ProgressInfo pi;
-  struct GNUNET_FS_DownloadContext *pos;
-
-  pos = dc->child_head;
-  while (pos != NULL)
-    {
-      if ( (pos->emsg == NULL) &&
-          (pos->completed < pos->length) )
-       return; /* not done yet */
-      if ( (pos->child_head != NULL) &&
-          (pos->has_finished != GNUNET_YES) )
-       return; /* not transitively done yet */
-      pos = pos->next;
-    }
-  dc->has_finished = GNUNET_YES;
-  /* signal completion */
-  pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
-  make_download_status (&pi, dc);
-  dc->client_info = dc->h->upcb (dc->h->upcb_cls,
-                                &pi);
-  if (dc->parent != NULL)
-    check_completed (dc->parent);  
-}
-
-
 /**
  * Iterator over entries in the pending requests in the 'active' map for the
  * reply that we just got.
@@ -1264,6 +1290,10 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc)
  * @param meta known metadata for the file (can be NULL)
  * @param filename where to store the file, maybe NULL (then no file is
  *        created on disk and data must be grabbed from the callbacks)
+ * @param tempname where to store temporary file data, not used if filename is non-NULL;
+ *        can be NULL (in which case we will pick a name if needed); the temporary file
+ *        may already exist, in which case we will try to use the data that is there and
+ *        if it is not what is desired, will overwrite it
  * @param offset at what offset should we start the download (typically 0)
  * @param length how many bytes should be downloaded starting at offset
  * @param anonymity anonymity level to use for the download
@@ -1278,6 +1308,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
                          const struct GNUNET_FS_Uri *uri,
                          const struct GNUNET_CONTAINER_MetaData *meta,
                          const char *filename,
+                         const char *tempname,
                          uint64_t offset,
                          uint64_t length,
                          uint32_t anonymity,
@@ -1352,7 +1383,12 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
   if ( (filename == NULL) &&
        (is_recursive_download (dc) ) )
-    dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");    
+    {
+      if (tempname != NULL)
+       dc->temp_filename = GNUNET_strdup (tempname);
+      else
+       dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");    
+    }
 
 #if DEBUG_DOWNLOAD
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,

diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c
index 0fd7b9c1d65ddf3728e002c4cc30d0185ac49332..6f36e20cb874c99d59422293dd9086229bfec5a9 100644 (file)
--- a/src/fs/fs_test_lib.c
+++ b/src/fs/fs_test_lib.c
@@ -608,7 +608,7 @@ GNUNET_FS_TEST_download (struct GNUNET_SCHEDULER_Handle *sched,
   daemon->download_seed = seed;  
   daemon->download_context = GNUNET_FS_download_start (daemon->fs,
                                                       uri,
-                                                      NULL,
+                                                      NULL, NULL,
                                                       NULL,
                                                       0,
                                                       size,

diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c
index 6f50922be124db239b03bf3a2e7e15af3c9bdfda..d9ab7543d8f91f0af35a9ff0fd0debabc4a398a9 100644 (file)
--- a/src/fs/gnunet-download.c
+++ b/src/fs/gnunet-download.c
@@ -221,7 +221,7 @@ run (void *cls,
   dc = GNUNET_FS_download_start (ctx,
                                 uri,
                                 NULL,
-                                filename,
+                                filename, NULL,
                                 0,
                                 GNUNET_FS_uri_chk_get_file_size (uri),
                                 anonymity,

diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c
index f33a1bd4a3885b02652cc7ffac9dded07b1a24c0..b06f0e965bfd1706b71d8cfbf765708e13ce267c 100644 (file)
--- a/src/fs/test_fs_download.c
+++ b/src/fs/test_fs_download.c
@@ -149,7 +149,7 @@ progress_cb (void *cls,
       download = GNUNET_FS_download_start (fs,
                                           event->value.publish.specifics.completed.chk_uri,
                                           NULL,
-                                          fn,
+                                          fn, NULL,
                                           0,
                                           FILESIZE,
                                           1,

diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index f24b3d8b7f2478eabac83ba8ae5c837e77bfc80e..6926b537e5d15dd74fc855d40a1f7007bc25841a 100644 (file)
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -2255,6 +2255,10 @@ enum GNUNET_FS_DownloadOptions
  * @param meta known metadata for the file (can be NULL)
  * @param filename where to store the file, maybe NULL (then no file is
  *        created on disk and data must be grabbed from the callbacks)
+ * @param tempname where to store temporary file data, not used if filename is non-NULL;
+ *        can be NULL (in which case we will pick a name if needed); the temporary file
+ *        may already exist, in which case we will try to use the data that is there and
+ *        if it is not what is desired, will overwrite it
  * @param offset at what offset should we start the download (typically 0)
  * @param length how many bytes should be downloaded starting at offset
  * @param anonymity anonymity level to use for the download
@@ -2269,6 +2273,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
                          const struct GNUNET_FS_Uri *uri,
                          const struct GNUNET_CONTAINER_MetaData *meta,
                          const char *filename,
+                         const char *tempname,
                          uint64_t offset,
                          uint64_t length,
                          uint32_t anonymity,