From: Christian Grothoff Date: Mon, 26 Apr 2010 15:25:27 +0000 (+0000) Subject: fully implement in-line files in directories X-Git-Tag: initial-import-from-subversion-38251~22005 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=e6674963348824eb7d8bdfdc0e2d655ca9773a40;p=oweals%2Fgnunet.git fully implement in-line files in directories --- diff --git a/TODO b/TODO index b5d21b828..a4393324c 100644 --- a/TODO +++ b/TODO @@ -1,13 +1,5 @@ 0.9.0pre1: -* MIGRATION [CG] - - on-demand encoding => move logic to block-library!? - - peer selection => how to consider latency/bw/etc? - - content transmission => how often the same block? - - how to select delay before next migration? - - testing * FS: [CG] - - migration (inbound) - - support for in-line files in directories (FIXME in fs_download) - bound parallelism (# fs downloads) - distinguish in performance tracking and event signalling between downloads that are actually running and those that are merely in the queue @@ -23,6 +15,15 @@ - [gnunet-service-fs.c:700]: member 'ConnectedPeer::last_client_replies_woff' is never used - GAP improvements: + active reply route caching design & implementation of service; gap extension! +* MIGRATION [CG] + - on-demand encoding => move logic to block-library!? + - peer selection => how to consider latency/bw/etc? + - content transmission => how often the same block? + - how to select delay before next migration? + - migration to us + - testing + - integrate with FS or not? (peer list, index/on-demand encoding, block code, + inbound priority assignment; all would be easier with tight integration!) * TBENCH: [MW] - good to have for transport/DV evaluation! * DV: [Nate] diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index b179c70dc..8194ad9f5 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -515,6 +515,125 @@ static void try_reconnect (struct GNUNET_FS_DownloadContext *dc); +/** + * We found an entry in a directory. Check if the respective child + * already exists and if not create the respective child download. + * + * @param cls the parent download + * @param filename name of the file in the directory + * @param uri URI of the file (CHK or LOC) + * @param meta meta data of the file + * @param length number of bytes in data + * @param data contents of the file (or NULL if they were not inlined) + */ +static void +trigger_recursive_download (void *cls, + const char *filename, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, + size_t length, + const void *data); + + +/** + * We're done downloading a directory. Open the file and + * trigger all of the (remaining) child downloads. + * + * @param dc context of download that just completed + */ +static void +full_recursive_download (struct GNUNET_FS_DownloadContext *dc) +{ + size_t size; + uint64_t size64; + void *data; + struct GNUNET_DISK_FileHandle *h; + struct GNUNET_DISK_MapHandle *m; + + size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri); + size = (size_t) size64; + if (size64 != (uint64_t) size) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n")); + return; + } + if (dc->filename != NULL) + { + h = GNUNET_DISK_file_open (dc->filename, + GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + } + else + { + GNUNET_assert (dc->temp_filename != NULL); + h = GNUNET_DISK_file_open (dc->temp_filename, + GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + } + if (h == NULL) + return; /* oops */ + data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size); + if (data == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Directory too large for system address space\n")); + } + else + { + GNUNET_FS_directory_list_contents (size, + data, + 0, + &trigger_recursive_download, + dc); + GNUNET_DISK_file_unmap (m); + } + GNUNET_DISK_file_close (h); + if (dc->filename == NULL) + { + if (0 != UNLINK (dc->temp_filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, + "unlink", + dc->temp_filename); + GNUNET_free (dc->temp_filename); + dc->temp_filename = NULL; + } +} + + +/** + * Check if all child-downloads have completed and + * if so, signal completion (and possibly recurse to + * parent). + */ +static void +check_completed (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_DownloadContext *pos; + + pos = dc->child_head; + while (pos != NULL) + { + if ( (pos->emsg == NULL) && + (pos->completed < pos->length) ) + return; /* not done yet */ + if ( (pos->child_head != NULL) && + (pos->has_finished != GNUNET_YES) ) + return; /* not transitively done yet */ + pos = pos->next; + } + dc->has_finished = GNUNET_YES; + /* signal completion */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; + make_download_status (&pi, dc); + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + if (dc->parent != NULL) + check_completed (dc->parent); +} + + /** * We found an entry in a directory. Check if the respective child * already exists and if not create the respective child download. @@ -537,6 +656,8 @@ trigger_recursive_download (void *cls, struct GNUNET_FS_DownloadContext *dc = cls; struct GNUNET_FS_DownloadContext *cpos; struct GNUNET_DISK_FileHandle *fh; + char *temp_name; + const char *real_name; char *fn; char *us; char *ext; @@ -632,45 +753,48 @@ trigger_recursive_download (void *cls, GNUNET_free_non_null (fn); return; } - - if (data != NULL) + + temp_name = NULL; + if ( (data != NULL) && + (GNUNET_FS_uri_chk_get_file_size (uri) == length) ) { - if (full_name != NULL) + if (full_name == NULL) { - fh = GNUNET_DISK_file_open (full_name, - GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE, - GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE); - if (fh == NULL) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, - "open", - full_name); - GNUNET_free (full_name); - GNUNET_free_non_null (fn); - return; - } - if (length != - GNUNET_DISK_file_write (fh, - data, - length)) - { - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, - "write", - full_name); - } - GNUNET_DISK_file_close (fh); + temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + real_name = temp_name; } else { - /* FIXME: generate 'progress' events and move to - instant completion! */ - GNUNET_break (0); // FIXME: not implemented + real_name = full_name; } + /* write to disk, then trigger normal download which will instantly progress to completion */ + fh = GNUNET_DISK_file_open (real_name, + GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE); + if (fh == NULL) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, + "open", + real_name); + GNUNET_free (full_name); + GNUNET_free_non_null (fn); + return; + } + if (length != + GNUNET_DISK_file_write (fh, + data, + length)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, + "write", + full_name); + } + GNUNET_DISK_file_close (fh); } GNUNET_FS_download_start (dc->h, uri, meta, - full_name, + full_name, temp_name, 0, GNUNET_FS_uri_chk_get_file_size (uri), dc->anonymity, @@ -678,109 +802,11 @@ trigger_recursive_download (void *cls, NULL, dc); GNUNET_free_non_null (full_name); + GNUNET_free_non_null (temp_name); GNUNET_free_non_null (fn); } -/** - * We're done downloading a directory. Open the file and - * trigger all of the (remaining) child downloads. - * - * @param dc context of download that just completed - */ -static void -full_recursive_download (struct GNUNET_FS_DownloadContext *dc) -{ - size_t size; - uint64_t size64; - void *data; - struct GNUNET_DISK_FileHandle *h; - struct GNUNET_DISK_MapHandle *m; - - size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri); - size = (size_t) size64; - if (size64 != (uint64_t) size) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n")); - return; - } - if (dc->filename != NULL) - { - h = GNUNET_DISK_file_open (dc->filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - } - else - { - GNUNET_assert (dc->temp_filename != NULL); - h = GNUNET_DISK_file_open (dc->temp_filename, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE); - } - if (h == NULL) - return; /* oops */ - data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size); - if (data == NULL) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("Directory too large for system address space\n")); - } - else - { - GNUNET_FS_directory_list_contents (size, - data, - 0, - &trigger_recursive_download, - dc); - GNUNET_DISK_file_unmap (m); - } - GNUNET_DISK_file_close (h); - if (dc->filename == NULL) - { - if (0 != UNLINK (dc->temp_filename)) - GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, - "unlink", - dc->temp_filename); - GNUNET_free (dc->temp_filename); - dc->temp_filename = NULL; - } -} - - -/** - * Check if all child-downloads have completed and - * if so, signal completion (and possibly recurse to - * parent). - */ -static void -check_completed (struct GNUNET_FS_DownloadContext *dc) -{ - struct GNUNET_FS_ProgressInfo pi; - struct GNUNET_FS_DownloadContext *pos; - - pos = dc->child_head; - while (pos != NULL) - { - if ( (pos->emsg == NULL) && - (pos->completed < pos->length) ) - return; /* not done yet */ - if ( (pos->child_head != NULL) && - (pos->has_finished != GNUNET_YES) ) - return; /* not transitively done yet */ - pos = pos->next; - } - dc->has_finished = GNUNET_YES; - /* signal completion */ - pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; - make_download_status (&pi, dc); - dc->client_info = dc->h->upcb (dc->h->upcb_cls, - &pi); - if (dc->parent != NULL) - check_completed (dc->parent); -} - - /** * Iterator over entries in the pending requests in the 'active' map for the * reply that we just got. @@ -1264,6 +1290,10 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc) * @param meta known metadata for the file (can be NULL) * @param filename where to store the file, maybe NULL (then no file is * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it * @param offset at what offset should we start the download (typically 0) * @param length how many bytes should be downloaded starting at offset * @param anonymity anonymity level to use for the download @@ -1278,6 +1308,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, + const char *tempname, uint64_t offset, uint64_t length, uint32_t anonymity, @@ -1352,7 +1383,12 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length)); if ( (filename == NULL) && (is_recursive_download (dc) ) ) - dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + { + if (tempname != NULL) + dc->temp_filename = GNUNET_strdup (tempname); + else + dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + } #if DEBUG_DOWNLOAD GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c index 0fd7b9c1d..6f36e20cb 100644 --- a/src/fs/fs_test_lib.c +++ b/src/fs/fs_test_lib.c @@ -608,7 +608,7 @@ GNUNET_FS_TEST_download (struct GNUNET_SCHEDULER_Handle *sched, daemon->download_seed = seed; daemon->download_context = GNUNET_FS_download_start (daemon->fs, uri, - NULL, + NULL, NULL, NULL, 0, size, diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c index 6f50922be..d9ab7543d 100644 --- a/src/fs/gnunet-download.c +++ b/src/fs/gnunet-download.c @@ -221,7 +221,7 @@ run (void *cls, dc = GNUNET_FS_download_start (ctx, uri, NULL, - filename, + filename, NULL, 0, GNUNET_FS_uri_chk_get_file_size (uri), anonymity, diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c index f33a1bd4a..b06f0e965 100644 --- a/src/fs/test_fs_download.c +++ b/src/fs/test_fs_download.c @@ -149,7 +149,7 @@ progress_cb (void *cls, download = GNUNET_FS_download_start (fs, event->value.publish.specifics.completed.chk_uri, NULL, - fn, + fn, NULL, 0, FILESIZE, 1, diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index f24b3d8b7..6926b537e 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -2255,6 +2255,10 @@ enum GNUNET_FS_DownloadOptions * @param meta known metadata for the file (can be NULL) * @param filename where to store the file, maybe NULL (then no file is * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it * @param offset at what offset should we start the download (typically 0) * @param length how many bytes should be downloaded starting at offset * @param anonymity anonymity level to use for the download @@ -2269,6 +2273,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, + const char *tempname, uint64_t offset, uint64_t length, uint32_t anonymity,