From 6cba55a63e41448c28555322f4ff97729804f9c8 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Mon, 3 May 2010 16:39:34 +0000 Subject: [PATCH] more fs stuff --- TODO | 17 ++- src/fs/fs.c | 242 ++++++++++++++++++++++++++++++++++++++++++- src/fs/fs.h | 29 ++++++ src/fs/fs_download.c | 26 ++--- 4 files changed, 298 insertions(+), 16 deletions(-) diff --git a/TODO b/TODO index df1836f06..9469382ce 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,24 @@ 0.9.0pre1: * FS: [CG] + - resume signalling for search/download must be recursive! - deserialization code (download) - serialization code (download) - - linking of downloads to searches + - linking of downloads to searches (expose opaque struct SearchResult; + allow starting download based on search result (new API): + => have meta data for instant completion! + => have URI + => linking of download to search + => expose link to search result in download events (including search result's + client-info pointer!) - generate SUSPEND events (publish, unindex, search, download) - actually call 'sync' functions (publish, unindex, search, download) - - persistence testing (publish, unindex) + - code review: + => refactor fs.c to join common code segments! + => document directory structure + => ensure all files & dirs are cleaned up! (at least during 'clean' runs) + - persistence testing (publish, unindex, search, download): + => need driver! + => schedule suspending tasks DURING event handler => good coverage! - gnunet-service-fs (hot-path routing, load-based routing, nitpicks) - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is never used diff --git a/src/fs/fs.c b/src/fs/fs.c index 6556fee18..053fd3dd0 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c @@ -1243,6 +1243,21 @@ GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) } +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) +{ + /* FIXME */ +} + + /** * Synchronize this search result with its mirror * on disk. Note that all internal FS-operations that change @@ -1699,6 +1714,167 @@ free_search_context (struct GNUNET_FS_SearchContext *sc) } +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + const char *serialization); + + +/** + * Function called with a filename of serialized sub-download + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_DownloadContext*' (parent) + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_subdownload (void *cls, + const char *filename) +{ + struct GNUNET_FS_DownloadContext *parent = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + deserialize_download (parent->h, + rh, + parent, + ser); + if (GNUNET_OK != + GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-download `%s': %s\n"), + ser, + emsg); + GNUNET_free (emsg); + } + GNUNET_free (ser); + return GNUNET_OK; +} + + +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + const char *serialization) +{ + struct GNUNET_FS_DownloadContext *dc; + struct GNUNET_FS_DownloadContext *dcc; + char pbuf[32]; + struct GNUNET_FS_ProgressInfo pi; + char *emsg; + char *uris; + char *dn; + + uris = NULL; + emsg = NULL; + dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); + dc->parent = parent; + dc->h = h; + dc->serialization = GNUNET_strdup (serialization); +#if 0 + /* FIXME */ + if ( (GNUNET_OK != + GNUNET_BIO_read_string (rh, "-uri", &uris, 10*1024)) || + (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + ( (GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) && + (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri)) ) || + (GNUNET_OK != + GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) || + (GNUNET_OK != + GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof (in_pause))) || + (GNUNET_OK != + GNUNET_BIO_read_int32 (rh, &sc->anonymity)) ) + goto cleanup; + /* FIXME: adjust start_time.value */ + sc->options = (enum GNUNET_FS_SearchOptions) options; + sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16); +#endif + GNUNET_snprintf (pbuf, + sizeof (pbuf), + "%s%s%s", + "subdownloads", + DIR_SEPARATOR_STR, + dc->serialization); + dn = get_serialization_file_name (h, pbuf, ""); + if (dn != NULL) + { + GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc); + GNUNET_free (dn); + } +#if 0 + if ('\0' == in_pause) + { + if (GNUNET_OK != + GNUNET_FS_search_start_searching_ (sc)) + goto cleanup; + } +#endif + if (0) + goto cleanup; + if (parent != NULL) + GNUNET_CONTAINER_DLL_insert (parent->child_head, + parent->child_tail, + dc); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; +#if 0 + pi.value.search.specifics.resume.message = sc->emsg; + pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; +#endif + GNUNET_FS_download_make_status_ (&pi, + dc); + dcc = dc->child_head; + while (NULL != dcc) + { + /* FIXME: wrong, need recursion! */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; +#if 0 + pi.value.search.specifics.resume.message = scc->emsg; + pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; +#endif + GNUNET_FS_download_make_status_ (&pi, + dcc); + dcc = dcc->next; + } +#if 0 + GNUNET_free (uris); +#endif + return; + cleanup: +#if 0 + GNUNET_free_non_null (emsg); + free_search_context (sc); +#endif + GNUNET_free_non_null (uris); +} + + /** * Deserialize a search. * @@ -1808,6 +1984,7 @@ deserialize_search (struct GNUNET_FS_Handle *h, scc = sc->child_head; while (NULL != scc) { + /* FIXME: wrong, need recursion! */ pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; pi.value.search.specifics.resume.message = scc->emsg; pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES; @@ -1864,7 +2041,7 @@ deserialize_search_file (void *cls, GNUNET_BIO_read_close (rh, &emsg)) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failure while resuming unindexing operation `%s': %s\n"), + _("Failure while resuming search operation `%s': %s\n"), filename, emsg); GNUNET_free (emsg); @@ -1891,6 +2068,67 @@ deserialize_search_master (struct GNUNET_FS_Handle *h) } +/** + * Function called with a filename of serialized download operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_download_file (void *cls, + const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (ser != NULL) + { + GNUNET_FS_remove_sync_file_ (h, "download", ser); + GNUNET_free (ser); + } + return GNUNET_OK; + } + deserialize_download (h, rh, NULL, ser); + GNUNET_free (ser); + if (GNUNET_OK != + GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming download operation `%s': %s\n"), + filename, + emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Deserialize information about pending download operations. + * + * @param h master context + */ +static void +deserialize_download_master (struct GNUNET_FS_Handle *h) +{ + char *dn; + + dn = get_serialization_file_name (h, "download", ""); + if (dn == NULL) + return; + GNUNET_DISK_directory_scan (dn, &deserialize_download_file, h); + GNUNET_free (dn); +} + + /** * Setup a connection to the file-sharing service. * @@ -1960,7 +2198,7 @@ GNUNET_FS_start (struct GNUNET_SCHEDULER_Handle *sched, function instead of these four... */ deserialize_publish (ret); deserialize_search_master (ret); - /* FIXME: deserialize downloads that are NOT part of searches */ + deserialize_download_master (ret); deserialize_unindex (ret); } return ret; diff --git a/src/fs/fs.h b/src/fs/fs.h index 338b38fc7..d3960c265 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -765,6 +765,19 @@ GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, const struct GNUNET_FS_FileInformation *p, uint64_t offset); + +/** + * Fill in all of the generic fields for a download event and call the + * callback. + * + * @param pi structure to fill in + * @param dc overall download context + */ +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc); + + /** * Fill in all of the generic fields for * an unindex event and call the callback. @@ -887,6 +900,16 @@ void GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key, struct SearchResult *sr); +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); /** * Master context for most FS operations. @@ -1432,6 +1455,12 @@ struct GNUNET_FS_DownloadContext */ char *emsg; + /** + * Random portion of filename we use for syncing state of this + * download. + */ + char *serialization; + /** * Where are we writing the data (name of the * file, can be NULL!). diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index 31984da08..468b255ee 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -144,9 +144,9 @@ compute_dblock_offset (uint64_t offset, * @param pi structure to fill in * @param dc overall download context */ -static void -make_download_status (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_DownloadContext *dc) +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc) { pi->value.download.dc = dc; pi->value.download.cctx @@ -631,7 +631,7 @@ check_completed (struct GNUNET_FS_DownloadContext *dc) dc->has_finished = GNUNET_YES; /* signal completion */ pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); if (dc->parent != NULL) check_completed (dc->parent); } @@ -856,7 +856,7 @@ process_result_with_request (void *cls, /* signal error */ pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; pi.value.download.specifics.error.message = dc->emsg; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); /* abort all pending requests */ if (NULL != dc->th) { @@ -936,7 +936,7 @@ process_result_with_request (void *cls, /* signal error */ pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; pi.value.download.specifics.error.message = emsg; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); /* abort all pending requests */ if (NULL != dc->th) { @@ -984,7 +984,7 @@ process_result_with_request (void *cls, pi.value.download.specifics.progress.offset = sm->offset; pi.value.download.specifics.progress.data_len = prc->size; pi.value.download.specifics.progress.depth = sm->depth; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); GNUNET_assert (dc->completed <= dc->length); if (dc->completed == dc->length) { @@ -1015,7 +1015,7 @@ process_result_with_request (void *cls, { /* signal completion */ pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); if (dc->parent != NULL) check_completed (dc->parent); } @@ -1314,7 +1314,7 @@ activate_fs_download (void *cls, dc, GNUNET_TIME_UNIT_FOREVER_REL); pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc); @@ -1351,7 +1351,7 @@ deactivate_fs_download (void *cls) dc->client = NULL; } pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); } @@ -1479,7 +1479,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, // FIXME: make persistent pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; pi.value.download.specifics.start.meta = meta; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); schedule_block_download (dc, &dc->uri->data.chk.chk, 0, @@ -1538,7 +1538,7 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, dc); pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; - make_download_status (&pi, dc); + GNUNET_FS_download_make_status_ (&pi, dc); if (GNUNET_SCHEDULER_NO_TASK != dc->task) GNUNET_SCHEDULER_cancel (dc->h->sched, dc->task); @@ -1570,6 +1570,8 @@ GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, dc->temp_filename); GNUNET_free (dc->temp_filename); } + /* FIXME: clean up serialization file itself! */ + GNUNET_free_non_null (dc->serialization); GNUNET_free (dc); } -- 2.25.1