From 0b90310c9a26ffa0e195759b7a5d5c07fa33f642 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 5 Sep 2009 20:40:29 +0000 Subject: [PATCH] towards having download --- src/fs/fs.h | 155 ++++++++++++++--- src/fs/fs_download.c | 386 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 508 insertions(+), 33 deletions(-) diff --git a/src/fs/fs.h b/src/fs/fs.h index ed925d9c8..3dc60a5dc 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -337,12 +337,12 @@ struct GNUNET_FS_FileInformation /** * Desired anonymity level. */ - unsigned int anonymity; + uint32_t anonymity; /** * Desired priority (for keeping the content in the DB). */ - unsigned int priority; + uint32_t priority; }; @@ -427,12 +427,6 @@ struct GNUNET_FS_PublishContext */ char *nuid; - /** - * ID of the task performing the upload. NO_TASK - * if the upload has completed. - */ - GNUNET_SCHEDULER_TaskIdentifier upload_task; - /** * Our own client handle for the FS service; * only briefly used when we start to index a @@ -440,6 +434,23 @@ struct GNUNET_FS_PublishContext */ struct GNUNET_CLIENT_Connection *client; + /** + * Current position in the file-tree for the + * upload. + */ + struct GNUNET_FS_FileInformation *fi_pos; + + /** + * Connection to the datastore service. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * ID of the task performing the upload. NO_TASK + * if the upload has completed. + */ + GNUNET_SCHEDULER_TaskIdentifier upload_task; + /** * Typically GNUNET_NO. Set to GNUNET_YES if * "upload_task" is GNUNET_SCHEDULER_NO_TASK @@ -460,17 +471,6 @@ struct GNUNET_FS_PublishContext */ enum GNUNET_FS_PublishOptions options; - /** - * Current position in the file-tree for the - * upload. - */ - struct GNUNET_FS_FileInformation *fi_pos; - - /** - * Connection to the datastore service. - */ - struct GNUNET_DATASTORE_Handle *dsh; - /** * Space reservation ID with datastore service * for this upload. @@ -764,11 +764,128 @@ struct GNUNET_FS_SearchContext }; +/** + * Information about an active download request. + */ +struct DownloadRequest +{ + /** + * While pending, we keep all download requests + * in a linked list. + */ + struct DownloadRequest *next; + + /** + * CHK for the request. + */ + struct ContentHashKey chk; + + /** + * Offset of the corresponding block. + */ + uint64_t offset; + + /** + * Depth of the corresponding block in the tree. + */ + unsigned int depth; + + /** + * Set if this request is currently in the linked list of pending + * requests. Needed in case we get a response for a request that we + * have not yet send (due to FS bug or two blocks with identical + * content); in this case, we would need to remove the block from + * the pending list (and need a fast way to check if the block is on + * it). + */ + int is_pending; + +}; + + /** * Context for controlling a download. */ struct GNUNET_FS_DownloadContext { + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Parent download (used when downloading files + * in directories). + */ + struct GNUNET_FS_DownloadContext *parent; + + /** + * Context kept for the client. + */ + void *client_info; + + /** + * URI that identifies the file that + * we are downloading. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Where are we writing the data (name of the + * file, can be NULL!). + */ + char *filename; + + /** + * Map of active requests (those waiting + * for a response). The key is the hash + * of the encryped block (aka query). + */ + struct GNUNET_CONTAINER_MultiHashMap *active; + + /** + * Linked list of pending requests. + */ + struct DownloadRequest *pending; + + /** + * ID of a task that is using this struct + * and that must be cancelled when the download + * is being stopped (if not GNUNET_SCHEDULER_NO_TASK). + * Used for the task that adds some artificial + * delay when trying to reconnect to the FS + * service. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * What is the first offset that we're interested + * in? + */ + uint64_t offset; + + /** + * How many bytes starting from offset are desired? + * This is NOT the overall length of the file! + */ + uint64_t length; + + /** + * Desired level of anonymity. + */ + uint32_t anonymity; + + /** + * Options for the download. + */ + enum GNUNET_FS_DownloadOptions options; + }; struct GNUNET_FS_Namespace diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index f1954ea89..06c4672a2 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008 Christian Grothoff (and other contributing authors) + (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -19,17 +19,288 @@ */ /** * @file fs/fs_download.c - * @brief DOWNLOAD helper methods (which do the real work). + * @brief download methods * @author Christian Grothoff + * + * TODO: + * - process replies + * - callback signaling + * - location URI suppport (can wait) + * - persistence (can wait) */ - #include "platform.h" +#include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "fs.h" #define DEBUG_DOWNLOAD GNUNET_YES +/** + * Schedule the download of the specified + * block in the tree. + * + * @param dc overall download this block belongs to + * @param chk content-hash-key of the block + * @param offset offset of the block in the file + * (for IBlocks, the offset is the lowest + * offset of any DBlock in the subtree under + * the IBlock) + * @param depth depth of the block, 0 is the root of the tree + */ +static void +schedule_block_download (struct GNUNET_FS_DownloadContext *dc, + const struct ContentHashKey *chk, + uint64_t offset, + unsigned int depth) +{ + struct DownloadRequest *sm; + + sm = GNUNET_malloc (sizeof (struct DownloadRequest)); + sm->chk = *chk; + sm->offset = offset; + sm->depth = depth; + sm->is_pending = GNUNET_YES; + sm->next = dc->pending; + dc->pending = sm; + GNUNET_CONTAINER_multihashmap_put (dc->active, + &chk->query, + sm, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); +} + + +/** + * We've lost our connection with the FS service. + * Re-establish it and re-transmit all of our + * pending requests. + * + * @param dc download context that is having trouble + */ +static void +try_reconnect (struct GNUNET_FS_DownloadContext *dc); + + +/** + * Process a search result. + * + * @param sc our search context + * @param type type of the result + * @param data the (encrypted) response + * @param size size of data + */ +static void +process_result (struct GNUNET_FS_DownloadContext *dc, + uint32_t type, + const void *data, + size_t size) +{ + GNUNET_HashCode query; + struct DownloadRequest *sm; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + char pt[size]; + + GNUNET_CRYPTO_hash (data, size, &query); + sm = GNUNET_CONTAINER_multihashmap_get (dc->active, + &query); + if (NULL == sm) + { + GNUNET_break (0); + return; + } + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (dc->active, + &query, + sm)); + GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv); + GNUNET_CRYPTO_aes_decrypt (data, + size, + &skey, + &iv, + pt); + // FIXME: save to disk + // FIXME: make persistent + // FIXME: call progress callback + // FIXME: trigger next block (if applicable) +} + + +/** + * Type of a function to call when we receive a message + * from the service. + * + * @param cls closure + * @param msg message received, NULL on timeout or fatal error + */ +static void +receive_results (void *cls, + const struct GNUNET_MessageHeader * msg) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + const struct ContentMessage *cm; + uint16_t msize; + + if ( (NULL == msg) || + (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) || + (ntohs (msg->size) <= sizeof (struct ContentMessage)) ) + { + try_reconnect (dc); + return; + } + msize = ntohs (msg->size); + cm = (const struct ContentMessage*) msg; + process_result (dc, + ntohl (cm->type), + &cm[1], + msize - sizeof (struct ContentMessage)); + /* continue receiving */ + GNUNET_CLIENT_receive (dc->client, + &receive_results, + dc, + GNUNET_TIME_UNIT_FOREVER_REL); +} + + + +/** + * We're ready to transmit a search request to the + * file-sharing service. Do it. If there is + * more than one request pending, try to send + * multiple or request another transmission. + * + * @param cls closure + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_download_request (void *cls, + size_t size, + void *buf) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + size_t msize; + struct SearchMessage *sm; + + if (NULL == buf) + { + try_reconnect (dc); + return 0; + } + GNUNET_assert (size >= sizeof (struct SearchMessage)); + msize = 0; + sm = buf; + while ( (dc->pending == NULL) && + (size > msize + sizeof (struct SearchMessage)) ) + { + memset (sm, 0, sizeof (struct SearchMessage)); + sm->header.size = htons (sizeof (struct SearchMessage)); + sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + sm->anonymity_level = htonl (dc->anonymity); + // FIXME: support 'loc' URIs (set sm->target) + sm->query = dc->pending->chk.query; + dc->pending->is_pending = GNUNET_NO; + dc->pending = dc->pending->next; + msize += sizeof (struct SearchMessage); + sm++; + } + return msize; +} + + +/** + * Reconnect to the FS service and transmit + * our queries NOW. + * + * @param cls our download context + * @param tc unused + */ +static void +do_reconnect (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_CLIENT_Connection *client; + + dc->task = GNUNET_SCHEDULER_NO_TASK; + client = GNUNET_CLIENT_connect (dc->h->sched, + "fs", + dc->h->cfg); + if (NULL == client) + { + try_reconnect (dc); + return; + } + dc->client = client; + GNUNET_CLIENT_notify_transmit_ready (client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &transmit_download_request, + dc); + GNUNET_CLIENT_receive (client, + &receive_results, + dc, + GNUNET_TIME_UNIT_FOREVER_REL); +} + + +/** + * Add entries that are not yet pending back to + * the pending list. + * + * @param cls our download context + * @param key unused + * @param entry entry of type "struct DownloadRequest" + * @return GNUNET_OK + */ +static int +retry_entry (void *cls, + const GNUNET_HashCode *key, + void *entry) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct DownloadRequest *dr = entry; + + if (! dr->is_pending) + { + dr->next = dc->pending; + dr->is_pending = GNUNET_YES; + dc->pending = entry; + } + return GNUNET_OK; +} + + +/** + * We've lost our connection with the FS service. + * Re-establish it and re-transmit all of our + * pending requests. + * + * @param dc download context that is having trouble + */ +static void +try_reconnect (struct GNUNET_FS_DownloadContext *dc) +{ + + if (NULL != dc->client) + { + GNUNET_CONTAINER_multihashmap_iterate (dc->active, + &retry_entry, + dc); + GNUNET_CLIENT_disconnect (dc->client); + dc->client = NULL; + } + dc->task + = GNUNET_SCHEDULER_add_delayed (dc->h->sched, + GNUNET_NO, + GNUNET_SCHEDULER_PRIORITY_IDLE, + GNUNET_SCHEDULER_NO_TASK, + GNUNET_TIME_UNIT_SECONDS, + &do_reconnect, + dc); +} + /** * Download parts of a file. Note that this will store @@ -50,9 +321,7 @@ * @param offset at what offset should we start the download (typically 0) * @param length how many bytes should be downloaded starting at offset * @param anonymity anonymity level to use for the download - * @param no_temporaries set to GNUNET_YES to disallow generation of temporary files - * @param recursive should this be a recursive download (useful for directories - * to automatically trigger download of files in the directories) + * @param options various options * @param parent parent download to associate this download with (use NULL * for top-level downloads; useful for manually-triggered recursive downloads) * @return context that can be used to control this download @@ -61,31 +330,120 @@ struct GNUNET_FS_DownloadContext * GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, const char *filename, - unsigned long long offset, - unsigned long long length, - unsigned int anonymity, - int no_temporaries, - int recursive, + uint64_t offset, + uint64_t length, + uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, struct GNUNET_FS_DownloadContext *parent) { - return NULL; + struct GNUNET_FS_DownloadContext *dc; + struct GNUNET_CLIENT_Connection *client; + + client = GNUNET_CLIENT_connect (h->sched, + "fs", + h->cfg); + if (NULL == client) + return NULL; + // FIXME: add support for "loc" URIs! + GNUNET_assert (GNUNET_FS_uri_test_chk (uri)); + if ( (dc->offset + dc->length < dc->offset) || + (dc->offset + dc->length > uri->data.chk.file_length) ) + { + GNUNET_break (0); + return NULL; + } + dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext)); + dc->h = h; + dc->client = client; + dc->parent = parent; + dc->uri = GNUNET_FS_uri_dup (uri); + dc->filename = (NULL == filename) ? NULL : GNUNET_strdup (filename); + dc->offset = offset; + dc->length = length; + dc->anonymity = anonymity; + dc->options = options; + dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE)); + // FIXME: make persistent + schedule_block_download (dc, + &dc->uri->data.chk.chk, + 0, + 0); + GNUNET_CLIENT_notify_transmit_ready (client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &transmit_download_request, + dc); + GNUNET_CLIENT_receive (client, + &receive_results, + dc, + GNUNET_TIME_UNIT_FOREVER_REL); + // FIXME: signal download start + return dc; +} + + +/** + * Free entries in the map. + * + * @param cls unused (NULL) + * @param key unused + * @param entry entry of type "struct DownloadRequest" which is freed + * @return GNUNET_OK + */ +static int +free_entry (void *cls, + const GNUNET_HashCode *key, + void *entry) +{ + GNUNET_free (entry); + return GNUNET_OK; } + /** * Stop a download (aborts if download is incomplete). * - * @param rm handle for the download + * @param dc handle for the download * @param do_delete delete files of incomplete downloads */ void -GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *rm, +GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete) { + // FIXME: make unpersistent + // FIXME: signal download end + + if (GNUNET_SCHEDULER_NO_TASK != dc->task) + GNUNET_SCHEDULER_cancel (dc->h->sched, + dc->task); + if (NULL != dc->client) + GNUNET_CLIENT_disconnect (dc->client); + GNUNET_CONTAINER_multihashmap_iterate (dc->active, + &free_entry, + NULL); + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + GNUNET_FS_uri_destroy (dc->uri); + GNUNET_free_non_null (dc->filename); + GNUNET_free (dc); } + + + + + + + + + + + + + + #if 0 /** -- 2.25.1