From: Christian Grothoff Date: Mon, 3 May 2010 21:11:14 +0000 (+0000) Subject: allow linking of downloads to search results X-Git-Tag: initial-import-from-subversion-38251~21905 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=4092583a7dcb0844d5957909ef81f6766b3f2603;p=oweals%2Fgnunet.git allow linking of downloads to search results --- diff --git a/TODO b/TODO index a460bf438..88d5d2877 100644 --- a/TODO +++ b/TODO @@ -1,14 +1,7 @@ 0.9.0pre1: * FS: [CG] - - generate SUSPEND events (publish, unindex, search, download) AND free memory! + - implement linking of downloads to searches in syncing (serialize/deserialize) - actually call 'sync' functions (publish, unindex, search, download) - - linking of downloads to searches (expose opaque struct SearchResult; - allow starting download based on search result (new API): - => have meta data for instant completion! - => have URI - => linking of download to search - => expose link to search result in download events (including search result's - client-info pointer!) - code review: => refactor fs.c to join common code segments! => document directory structure (and use #define's for the directory names!) @@ -16,6 +9,9 @@ - persistence testing (publish, unindex, search, download): => need driver! => schedule suspending tasks DURING event handler => good coverage! + - track top-level operations in FS_Handle (needed for SUSPEND signalling) + - generate SUSPEND events (publish, unindex, search, download) AND free memory! + => test SUSPEND events - gnunet-service-fs (hot-path routing, load-based routing, nitpicks) - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is never used diff --git a/src/fs/fs.c b/src/fs/fs.c index 38b48ff8b..f0753eee2 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c @@ -1486,7 +1486,7 @@ GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) */ void GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key, - struct SearchResult *sr) + struct GNUNET_FS_SearchResult *sr) { struct GNUNET_BIO_WriteHandle *wh; char *uris; @@ -1779,7 +1779,7 @@ deserialize_search_result (void *cls, char *uris; char *emsg; struct GNUNET_BIO_ReadHandle *rh; - struct SearchResult *sr; + struct GNUNET_FS_SearchResult *sr; GNUNET_HashCode key; ser = get_serialization_short_name (filename); @@ -1801,7 +1801,7 @@ deserialize_search_result (void *cls, } emsg = NULL; uris = NULL; - sr = GNUNET_malloc (sizeof (struct SearchResult)); + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); sr->serialization = ser; if ( (GNUNET_OK != GNUNET_BIO_read_string (rh, "result-uri", &uris, 10*1024)) || @@ -1846,7 +1846,7 @@ deserialize_search_result (void *cls, * * @param cls closure, the 'struct GNUNET_FS_SearchContext' * @param key current key code - * @param value value in the hash map, the 'struct SearchResult' + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' * @return GNUNET_YES (we should continue to iterate) */ static int @@ -1856,13 +1856,14 @@ signal_result_resume (void *cls, { struct GNUNET_FS_SearchContext *sc = cls; struct GNUNET_FS_ProgressInfo pi; - struct SearchResult *sr = value; + struct GNUNET_FS_SearchResult *sr = value; if (0 == sr->mandatory_missing) { pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT; pi.value.search.specifics.resume_result.meta = sr->meta; pi.value.search.specifics.resume_result.uri = sr->uri; + pi.value.search.specifics.resume_result.result = sr; pi.value.search.specifics.resume_result.availability_rank = 2*sr->availability_success - sr->availability_trials; pi.value.search.specifics.resume_result.availability_certainty = sr->availability_trials; pi.value.search.specifics.resume_result.applicability_rank = sr->optional_support; @@ -1879,7 +1880,7 @@ signal_result_resume (void *cls, * * @param cls closure, the 'struct GNUNET_FS_SearchContext' * @param key current key code - * @param value value in the hash map, the 'struct SearchResult' + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' * @return GNUNET_YES (we should continue to iterate) */ static int @@ -1887,7 +1888,7 @@ free_result (void *cls, const GNUNET_HashCode * key, void *value) { - struct SearchResult *sr = value; + struct GNUNET_FS_SearchResult *sr = value; GNUNET_CONTAINER_meta_data_destroy (sr->meta); GNUNET_FS_uri_destroy (sr->uri); diff --git a/src/fs/fs.h b/src/fs/fs.h index f3b0a8624..7b30367ea 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -544,7 +544,7 @@ struct GNUNET_FS_QueueEntry /** * Information we store for each search result. */ -struct SearchResult +struct GNUNET_FS_SearchResult { /** @@ -573,6 +573,14 @@ struct SearchResult */ struct GNUNET_FS_DownloadContext *probe_ctx; + /** + * ID of an associated download based on this search result (or + * NULL for none). + * + * FIXME: not yet serialized. + */ + struct GNUNET_FS_DownloadContext *download; + /** * Name under which this search result is stored on disk. */ @@ -837,7 +845,7 @@ GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); * @param sr the search result */ void -GNUNET_FS_search_start_probe_ (struct SearchResult *sr); +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); /** * Remove serialization/deserialization file from disk. @@ -907,7 +915,7 @@ GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); */ void GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key, - struct SearchResult *sr); + struct GNUNET_FS_SearchResult *sr); /** * Synchronize this download struct with its mirror @@ -1237,7 +1245,7 @@ struct SearchRequestEntry GNUNET_HashCode query; /** - * Map that contains a "struct SearchResult" for each result that + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that * was found under this keyword. Note that the entries will point * to the same locations as those in the master result map (in * "struct GNUNET_FS_SearchContext"), so they should not be freed. @@ -1321,7 +1329,7 @@ struct GNUNET_FS_SearchContext char *emsg; /** - * Map that contains a "struct SearchResult" for each result that + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that * was found in the search. The key for each entry is the XOR of * the key and query in the CHK URI (as a unique identifier for the * search result). @@ -1424,6 +1432,14 @@ struct GNUNET_FS_DownloadContext */ struct GNUNET_FS_DownloadContext *parent; + /** + * Associated search (used when downloading files + * based on search results), or NULL for none. + * + * FIXME: not yet serialized + */ + struct GNUNET_FS_SearchResult *search; + /** * Head of list of child downloads. */ diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index 25ae2002a..6390539bc 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -153,6 +153,8 @@ GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, = dc->client_info; pi->value.download.pctx = (dc->parent == NULL) ? NULL : dc->parent->client_info; + pi->value.download.sctx + = (dc->search == NULL) ? NULL : dc->search->client_info; pi->value.download.uri = dc->uri; pi->value.download.filename @@ -1434,7 +1436,6 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, GNUNET_break (0); return NULL; } - // FIXME: add support for "loc" URIs! #if DEBUG_DOWNLOAD GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting download `%s' of %llu bytes\n", @@ -1462,7 +1463,10 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, &dc->old_file_size, GNUNET_YES); } - // FIXME: set "dc->target" for LOC uris! + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, + &dc->target)); dc->offset = offset; dc->length = length; dc->anonymity = anonymity; @@ -1496,6 +1500,125 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, } +/** + * Download parts of a file based on a search result. The download + * will be associated with the search result (and the association + * will be preserved when serializing/deserializing the state). + * If the search is stopped, the download will not be aborted but + * be 'promoted' to a stand-alone download. + * + * As with the other download function, this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries).

+ * + * The given range can be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param sr the search result to use for the download (determines uri and + * meta data and associations) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various download options + * @param cctx initial value for the client context for this download + * @return context that can be used to control this download + */ +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_SearchResult *sr, + const char *filename, + const char *tempname, + uint64_t offset, + uint64_t length, + uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, + void *cctx) +{ + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_DownloadContext *dc; + + if (sr->download != NULL) + { + GNUNET_break (0); + return NULL; + } + GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri)); + if ( (offset + length < offset) || + (offset + length > sr->uri->data.chk.file_length) ) + { + GNUNET_break (0); + return NULL; + } +#if DEBUG_DOWNLOAD + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting download `%s' of %llu bytes\n", + filename, + (unsigned long long) length); +#endif + dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext)); + dc->h = h; + dc->search = sr; + sr->download = dc; + dc->uri = GNUNET_FS_uri_dup (sr->uri); + dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta); + dc->client_info = cctx; + dc->start_time = GNUNET_TIME_absolute_get (); + if (NULL != filename) + { + dc->filename = GNUNET_strdup (filename); + if (GNUNET_YES == GNUNET_DISK_file_test (filename)) + GNUNET_DISK_file_size (filename, + &dc->old_file_size, + GNUNET_YES); + } + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, + &dc->target)); + dc->offset = offset; + dc->length = length; + dc->anonymity = anonymity; + dc->options = options; + dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE)); + dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length)); + if ( (filename == NULL) && + (is_recursive_download (dc) ) ) + { + if (tempname != NULL) + dc->temp_filename = GNUNET_strdup (tempname); + else + dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + } + +#if DEBUG_DOWNLOAD + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Download tree has depth %u\n", + dc->treedepth); +#endif + // FIXME: make persistent + pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; + pi.value.download.specifics.start.meta = dc->meta; + GNUNET_FS_download_make_status_ (&pi, dc); + schedule_block_download (dc, + &dc->uri->data.chk.chk, + 0, + 1 /* 0 == CHK, 1 == top */); + GNUNET_FS_download_start_downloading_ (dc); + return dc; +} + + /** * Start the downloading process (by entering the queue). * @@ -1542,6 +1665,11 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, { struct GNUNET_FS_ProgressInfo pi; + if (dc->search != NULL) + { + dc->search->download = NULL; + dc->search = NULL; + } if (dc->job_queue != NULL) { GNUNET_FS_dequeue_ (dc->job_queue); diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index a9670cb43..611a089c4 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c @@ -73,7 +73,7 @@ GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, * * @param cls points to the URI we check against * @param key not used - * @param value a "struct SearchResult" who's URI we + * @param value a "struct GNUNET_FS_SearchResult" who's URI we * should compare with * @return GNUNET_SYSERR if the result is present, * GNUNET_OK otherwise @@ -84,7 +84,7 @@ test_result_present (void *cls, void *value) { const struct GNUNET_FS_Uri *uri = cls; - struct SearchResult *sr = value; + struct GNUNET_FS_SearchResult *sr = value; if (GNUNET_FS_uri_test_equal (uri, sr->uri)) @@ -102,13 +102,14 @@ test_result_present (void *cls, */ static void notify_client_chk_result (struct GNUNET_FS_SearchContext *sc, - struct SearchResult *sr) + struct GNUNET_FS_SearchResult *sr) { struct GNUNET_FS_ProgressInfo pi; pi.status = GNUNET_FS_STATUS_SEARCH_RESULT; pi.value.search.specifics.result.meta = sr->meta; pi.value.search.specifics.result.uri = sr->uri; + pi.value.search.specifics.result.result = sr; sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); } @@ -122,7 +123,7 @@ notify_client_chk_result (struct GNUNET_FS_SearchContext *sc, */ static void notify_client_chk_update (struct GNUNET_FS_SearchContext *sc, - struct SearchResult *sr) + struct GNUNET_FS_SearchResult *sr) { struct GNUNET_FS_ProgressInfo pi; @@ -154,7 +155,7 @@ struct GetResultContext * Where to store a pointer to the search * result struct if we found a match. */ - struct SearchResult *sr; + struct GNUNET_FS_SearchResult *sr; }; @@ -164,7 +165,7 @@ struct GetResultContext * * @param cls a "struct GetResultContext" * @param key not used - * @param value a "struct SearchResult" who's URI we + * @param value a "struct GNUNET_FS_SearchResult" who's URI we * should compare with * @return GNUNET_OK */ @@ -174,7 +175,7 @@ get_result_present (void *cls, void *value) { struct GetResultContext *grc = cls; - struct SearchResult *sr = value; + struct GNUNET_FS_SearchResult *sr = value; if (GNUNET_FS_uri_test_equal (grc->uri, sr->uri)) @@ -188,7 +189,7 @@ get_result_present (void *cls, * probe. */ static void -signal_probe_result (struct SearchResult *sr) +signal_probe_result (struct GNUNET_FS_SearchResult *sr) { struct GNUNET_FS_ProgressInfo pi; @@ -207,14 +208,14 @@ signal_probe_result (struct SearchResult *sr) /** * Handle the case where we have failed to receive a response for our probe. * - * @param cls our 'struct SearchResult*' + * @param cls our 'struct GNUNET_FS_SearchResult*' * @param tc scheduler context */ static void probe_failure_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - struct SearchResult *sr = cls; + struct GNUNET_FS_SearchResult *sr = cls; sr->availability_trials++; signal_probe_result (sr); } @@ -223,14 +224,14 @@ probe_failure_handler (void *cls, /** * Handle the case where we have gotten a response for our probe. * - * @param cls our 'struct SearchResult*' + * @param cls our 'struct GNUNET_FS_SearchResult*' * @param tc scheduler context */ static void probe_success_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - struct SearchResult *sr = cls; + struct GNUNET_FS_SearchResult *sr = cls; sr->availability_trials++; sr->availability_success++; signal_probe_result (sr); @@ -256,7 +257,7 @@ void* GNUNET_FS_search_probe_progress_ (void *cls, const struct GNUNET_FS_ProgressInfo *info) { - struct SearchResult *sr = info->value.download.cctx; + struct GNUNET_FS_SearchResult *sr = info->value.download.cctx; struct GNUNET_TIME_Relative dur; switch (info->status) @@ -336,7 +337,7 @@ GNUNET_FS_search_probe_progress_ (void *cls, * @param sr the search result */ void -GNUNET_FS_search_start_probe_ (struct SearchResult *sr) +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr) { uint64_t off; uint64_t len; @@ -391,7 +392,7 @@ process_ksk_result (struct GNUNET_FS_SearchContext *sc, const struct GNUNET_CONTAINER_MetaData *meta) { GNUNET_HashCode key; - struct SearchResult *sr; + struct GNUNET_FS_SearchResult *sr; struct GetResultContext grc; int is_new; @@ -414,7 +415,7 @@ process_ksk_result (struct GNUNET_FS_SearchContext *sc, is_new = (NULL == sr) || (sr->mandatory_missing > 0); if (NULL == sr) { - sr = GNUNET_malloc (sizeof (struct SearchResult)); + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); sr->sc = sc; sr->uri = GNUNET_FS_uri_dup (uri); sr->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); @@ -481,7 +482,7 @@ process_sks_result (struct GNUNET_FS_SearchContext *sc, { struct GNUNET_FS_Uri uu; GNUNET_HashCode key; - struct SearchResult *sr; + struct GNUNET_FS_SearchResult *sr; /* check if new */ GNUNET_FS_uri_to_key (uri, &key); @@ -494,7 +495,7 @@ process_sks_result (struct GNUNET_FS_SearchContext *sc, &test_result_present, (void*) uri)) return; /* duplicate result */ - sr = GNUNET_malloc (sizeof (struct SearchResult)); + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); sr->sc = sc; sr->uri = GNUNET_FS_uri_dup (uri); sr->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); @@ -1243,9 +1244,17 @@ search_result_free (void *cls, struct GNUNET_FS_SearchContext *sc = cls; struct GNUNET_FS_Handle *h = sc->h; char pbuf[32]; - struct SearchResult *sr = value; + struct GNUNET_FS_SearchResult *sr = value; struct GNUNET_FS_ProgressInfo pi; + if (NULL != sr->download) + { + sr->download->search = NULL; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT; + GNUNET_FS_download_make_status_ (&pi, + sr->download); + sr->download = NULL; + } pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED; pi.value.search.specifics.result_stopped.cctx = sr->client_info; pi.value.search.specifics.result_stopped.meta = sr->meta; diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index 1b2e9b525..0645a11c4 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -568,6 +568,14 @@ enum GNUNET_FS_Status */ GNUNET_FS_STATUS_DOWNLOAD_INACTIVE, + /** + * Notification that this download is no longer part of a + * recursive download or search but now a 'stand-alone' + * download (and may thus need to be moved in the GUI + * into a different category). + */ + GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT, + /** * First event generated when a client requests * a search to begin or when a namespace result @@ -704,6 +712,14 @@ struct GNUNET_FS_UnindexContext; struct GNUNET_FS_SearchContext; +/** + * Result from a search. Opaque handle to refer to the search + * (typically used when starting a download associated with the search + * result). + */ +struct GNUNET_FS_SearchResult; + + /** * Context for controlling a download. */ @@ -900,6 +916,14 @@ struct GNUNET_FS_ProgressInfo * (if this is a file in a directory or a subdirectory). */ void *pctx; + + /** + * Client context pointer for the associated search operation + * (specifically, context pointer for the specific search + * result, not the overall search); only set if this + * download was started from a search result. + */ + void *sctx; /** * URI used for this download. @@ -1092,6 +1116,11 @@ struct GNUNET_FS_ProgressInfo */ const struct GNUNET_FS_Uri *uri; + /** + * Handle to the result (for starting downloads). + */ + struct GNUNET_FS_SearchResult *result; + } result; /** @@ -1110,6 +1139,11 @@ struct GNUNET_FS_ProgressInfo */ const struct GNUNET_FS_Uri *uri; + /** + * Handle to the result (for starting downloads). + */ + struct GNUNET_FS_SearchResult *result; + /** * Current availability rank (negative: * unavailable, positive: available) @@ -2336,6 +2370,52 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, struct GNUNET_FS_DownloadContext *parent); +/** + * Download parts of a file based on a search result. The download + * will be associated with the search result (and the association + * will be preserved when serializing/deserializing the state). + * If the search is stopped, the download will not be aborted but + * be 'promoted' to a stand-alone download. + * + * As with the other download function, this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries).

+ * + * The given range can be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param sr the search result to use for the download (determines uri and + * meta data and associations) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various download options + * @param cctx initial value for the client context for this download + * @return context that can be used to control this download + */ +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_SearchResult *sr, + const char *filename, + const char *tempname, + uint64_t offset, + uint64_t length, + uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, + void *cctx); + + /** * Stop a download (aborts if download is incomplete). *