From: Christian Grothoff Date: Sun, 6 Sep 2009 12:22:50 +0000 (+0000) Subject: work on downloading X-Git-Tag: initial-import-from-subversion-38251~23520 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=c263f813925949b97bfd899e174eefe4ecccced8;p=oweals%2Fgnunet.git work on downloading --- diff --git a/TODO b/TODO index 5b3b4d48a..24c12a1bf 100644 --- a/TODO +++ b/TODO @@ -13,7 +13,7 @@ away), in order in which they will likely be done: * UPNP -PHASE #2: (Goal: recover basic file-sharing functionality) +Urgent items (before announcing ng.gnunet.org): * TESTING (needed for DV, DHT, Topology) - implement library for local testing + modify configuration to allow controlling @@ -37,10 +37,10 @@ PHASE #2: (Goal: recover basic file-sharing functionality) - implement testcases * FS (basic anonymous FS only) - implement basic FS library - - download API - - gnunet-search (binary) - - gnunet-download (binary) - - API FIX: change type for anonymity/priority from unsigned int to uint32_t! + - gnunet-publish (progress CB, shutdown, URI args) + - gnunet-unindex (progress CB, shutdown, error checking) + - gnunet-search (start search, progress CB, shutdown, options) + - gnunet-download (start download, progress CB, shutdown, options) - design network structs (P2P) + query + response @@ -57,21 +57,18 @@ PHASE #2: (Goal: recover basic file-sharing functionality) ~ download ~ search ~ unindex - ~ namespaces - ~ collection + directory API * new webpage - - lcov - - buildbot - - migrate Mantis? - - download links on Drupal? - - run peer - - configure hostlist - - install on proper server + - prevent? + - migrate Mantis!? + - Doxygen generation + - run peer => have a 0.9.x hostlist + - improve basic documentation (configure, dependencies, what works, etc.) + - write minimal welcome/introduction to 0.9.x-development/webpage/status => Deploy(able) development network -PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009] +0.9.0pre0: * Module features to implement: - advanced FS API parts + namespaces @@ -80,8 +77,9 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009] + collection + location URIs (publish, search, download) + persistence support (publish, unindex, search, download) - + datastore reservation (publishing) + indexing: index-failure-cleanup + + download: management/bounding of parallel downloads (for recursive dl.) + + datastore reservation (publishing) - implement adv. FS testcases + insert: sblocks, loc uris + download: loc uris @@ -103,6 +101,9 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009] - testing (RC-pre0) + implement library for distributed testing [Nate] + implement testcases for distributed testing [Nate] + +0.9.0pre1: +* Module features to implement: - tbench (RC-pre1) + good to have for DV evaluation! - DV (RC-pre1) @@ -112,6 +113,12 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009] + implement DV transport plugin [Nate & CG] + implement testcases [Nate] + implement performance tests [Nate] +* GUIs to implement: + - gtk (RC-pre1) + + how to integrate scheduler with GTK event loop! + +0.9.0pre2: +* Module features to implement: - tracekit (RC-pre2) + good to have for DV/DHT evaluation! - DHT (RC-pre2) @@ -120,25 +127,21 @@ PHASE #3: (Goal: ready for pre-release) [completion-goal: end of 2009] + implement testcases + implement performance tests * GUIs to implement: - - gtk (RC-pre1) - + how to integrate scheduler with GTK event loop! - fuse (RC-pre2) - - qt (RC-pre3) - + see discussions @ FISL about integration with event loop! * Plugins to implement: - UDP backend (RC-pre2) + Fragmentation library + actual plugin - HTTP backend (RC-pre2) -* Determine RC bugs and fix those! - -=> PRE-RELEASE +0.9.0pre3: +* GUIs to implement: + - qt (RC-pre3) + + see discussions @ FISL about integration with event loop! +* Determine RC bugs and fix those! -PHASE #4: [completion-goal: mid 2010] +0.9.0: * Documentation - - Doxygen generation - - lcov generation - update man pages - update webpage documentation * new webpage: @@ -161,13 +164,10 @@ PHASE #4: [completion-goal: mid 2010] - vpn * Determine RC bugs and fix those! -=> 0.9.0 RELEASE - - - -Post 0.9.0 features: -* SMTP transport backend -* HTTPS transport backend - - improved HTTPS support in MHD - - actual plugin +0.9.x: +* Plugins to implement: + - SMTP transport backend + - HTTPS transport backend + + improved HTTPS support in MHD + + actual plugin diff --git a/src/Makefile.am b/src/Makefile.am index 9b3aa46ab..7e92437c6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,5 +23,6 @@ SUBDIRS = \ core \ testing \ $(HOSTLIST_DIR) \ - topology + topology \ + fs diff --git a/src/fs/fs.h b/src/fs/fs.h index 50090c6fd..70e02360c 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -836,6 +836,16 @@ struct GNUNET_FS_DownloadContext */ struct GNUNET_FS_Uri *uri; + /** + * Known meta-data for the file (can be NULL). + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Error message, NULL if we're doing OK. + */ + char *emsg; + /** * Where are we writing the data (name of the * file, can be NULL!). diff --git a/src/fs/fs_collection.c b/src/fs/fs_collection.c index 01c7e3298..f94ef250d 100644 --- a/src/fs/fs_collection.c +++ b/src/fs/fs_collection.c @@ -139,12 +139,12 @@ typedef struct CollectionData /** * Anonymity level for the collection. (NBO) */ - unsigned int anonymityLevel; + uint32_t anonymityLevel; /** * Priority of the collection (NBO). */ - unsigned int priority; + uint32_t priority; /** * Has this collection changed since the last publication? @@ -465,8 +465,8 @@ GNUNET_CO_done () * is destroyed (i.e. on exit from the UI). */ int -GNUNET_CO_collection_start (unsigned int anonymityLevel, - unsigned int prio, +GNUNET_CO_collection_start (uint32_t anonymityLevel, + uint32_t prio, const struct GNUNET_MetaData *meta) { struct GNUNET_ECRS_URI *advertisement; diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index c5bfec52c..145e16818 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -23,10 +23,11 @@ * @author Christian Grothoff * * TODO: - * - offset calculations - * - callback signaling - * - check if blocks exist already (can wait) - * - location URI suppport (can wait) + * - location URI suppport (can wait, easy) + * - check if blocks exist already (can wait, easy) + * - handle recursive downloads (need directory & + * fs-level download-parallelism management, can wait) + * - check if iblocks can be computed from existing blocks (can wait, hard) * - persistence (can wait) */ #include "platform.h" @@ -46,6 +47,7 @@ * to the DBLOCKS) and its depth, return the * offset where we would store this block * in the file. + * * @param fsize overall file size * @param off offset of the block in the file @@ -58,29 +60,48 @@ */ static uint64_t compute_disk_offset (uint64_t fsize, - uint64_t off, - unsigned int depth, - unsigned int treedepth) + uint64_t off, + unsigned int depth, + unsigned int treedepth) { + unsigned int i; + uint64_t lsize; /* what is the size of all IBlocks for level "i"? */ + uint64_t loff; /* where do IBlocks for level "i" start? */ + unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */ + if (depth == treedepth) return off; - return 42; // FIXME + /* first IBlocks start at the end of file, rounded up + to full DBLOCK_SIZE */ + loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE; + lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey); + GNUNET_assert (0 == (off % DBLOCK_SIZE)); + ioff = (off / DBLOCK_SIZE); + for (i=treedepth-1;i>depth;i--) + { + loff += lsize; + lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE; + GNUNET_assert (lsize > 0); + GNUNET_assert (0 == (ioff % CHK_PER_INODE)); + ioff /= CHK_PER_INODE; + } + return loff + ioff * sizeof (struct ContentHashKey); } + /** - * Given a file of the specified treedepth and - * a block at the given offset and depth, - * calculate the offset for the CHK at - * the given index. + * Given a file of the specified treedepth and a block at the given + * offset and depth, calculate the offset for the CHK at the given + * index. * * @param offset the offset of the first * DBLOCK in the subtree of the * identified IBLOCK * @param depth the depth of the IBLOCK in the tree * @param treedepth overall depth of the tree - * @param i which CHK in the IBLOCK are we + * @param k which CHK in the IBLOCK are we * talking about - * @return offset if i=0, otherwise an appropriately + * @return offset if k=0, otherwise an appropriately * larger value (i.e., if depth = treedepth-1, * the returned value should be offset+DBLOCK_SIZE) */ @@ -88,12 +109,47 @@ static uint64_t compute_dblock_offset (uint64_t offset, unsigned int depth, unsigned int treedepth, - unsigned int i) + unsigned int k) { - GNUNET_assert (depth < treedepth); - if (i == 0) + unsigned int i; + uint64_t lsize; /* what is the size of the sum of all DBlocks + that a CHK at level i corresponds to? */ + + if (depth == treedepth) return offset; - return 42; // FIXME + lsize = DBLOCK_SIZE; + for (i=treedepth-1;i>depth;i--) + lsize *= CHK_PER_INODE; + return offset + i * lsize; +} + + +/** + * Fill in all of the generic fields for + * a download event. + * + * @param pc structure to fill in + * @param dc overall download context + */ +static void +make_download_status (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc) +{ + pi->value.download.dc = dc; + pi->value.download.cctx + = dc->client_info; + pi->value.download.pctx + = (dc->parent == NULL) ? NULL : dc->parent->client_info; + pi->value.download.uri + = dc->uri; + pi->value.download.length + = dc->length; + pi->value.download.duration + = GNUNET_TIME_absolute_get_duration (dc->start_time); + pi->value.download.completed + = dc->completed; + pi->value.download.anonymity + = dc->anonymity; } @@ -166,6 +222,50 @@ static void try_reconnect (struct GNUNET_FS_DownloadContext *dc); +/** + * Compute how many bytes of data should be stored in + * the specified node. + * + * @param fsize overall file size + * @param off offset of the node + * @param depth depth of the node + * @return number of bytes stored in this node + */ +static size_t +calculate_block_size (uint64_t fsize, + unsigned int totaldepth, + uint64_t offset, + unsigned int depth) +{ + unsigned int i; + size_t ret; + uint64_t rsize; + uint64_t epos; + unsigned int chks; + + GNUNET_assert (offset < fsize); + if (depth == totaldepth) + { + ret = DBLOCK_SIZE; + if (offset + ret > fsize) + ret = (size_t) (fsize - offset); + return ret; + } + + rsize = DBLOCK_SIZE; + for (i = totaldepth-1; i > depth; i--) + rsize *= CHK_PER_INODE; + epos = offset + rsize * CHK_PER_INODE; + GNUNET_assert (epos > offset); + if (epos > fsize) + epos = fsize; + /* round up when computing #CHKs in our IBlock */ + chks = (epos - offset + rsize - 1) / rsize; + GNUNET_assert (chks <= CHK_PER_INODE); + return chks * sizeof (struct ContentHashKey); +} + + /** * Process a search result. * @@ -180,6 +280,7 @@ process_result (struct GNUNET_FS_DownloadContext *dc, const void *data, size_t size) { + struct GNUNET_FS_ProgressInfo pi; GNUNET_HashCode query; struct DownloadRequest *sm; struct GNUNET_CRYPTO_AesSessionKey skey; @@ -189,8 +290,8 @@ process_result (struct GNUNET_FS_DownloadContext *dc, size_t app; unsigned int i; struct ContentHashKey *chk; + char *emsg; - // FIXME: check that size is as big as expected, otherwise ignore!!! GNUNET_CRYPTO_hash (data, size, &query); sm = GNUNET_CONTAINER_multihashmap_get (dc->active, &query); @@ -199,6 +300,23 @@ process_result (struct GNUNET_FS_DownloadContext *dc, GNUNET_break (0); return; } + if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length), + dc->treedepth, + sm->offset, + sm->depth)) + { + dc->emsg = GNUNET_strdup ("Internal error or bogus download URI"); + /* signal error */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + make_download_status (&pi, dc); + pi.value.download.specifics.error.message = dc->emsg; + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + /* abort all pending requests */ + GNUNET_CLIENT_disconnect (dc->client); + dc->client = NULL; + return; + } GNUNET_assert (GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove (dc->active, &query, @@ -218,16 +336,44 @@ process_result (struct GNUNET_FS_DownloadContext *dc, sm->offset, sm->depth, dc->treedepth); - GNUNET_assert (off != - GNUNET_DISK_file_seek (dc->handle, - off, - GNUNET_DISK_SEEK_SET) ); - GNUNET_DISK_file_write (dc->handle, - pt, - size); - } - // FIXME: make persistent + emsg = NULL; + if ( (off != + GNUNET_DISK_file_seek (dc->handle, + off, + GNUNET_DISK_SEEK_SET) ) ) + GNUNET_asprintf (&emsg, + _("Failed to seek to offset %llu in file `%s': %s\n"), + (unsigned long long) off, + dc->filename, + STRERROR (errno)); + else if (size != + GNUNET_DISK_file_write (dc->handle, + pt, + size)) + GNUNET_asprintf (&emsg, + _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"), + (unsigned int) size, + (unsigned long long) off, + dc->filename, + STRERROR (errno)); + if (NULL != emsg) + { + dc->emsg = emsg; + // FIXME: make persistent + + /* signal error */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + make_download_status (&pi, dc); + pi.value.download.specifics.error.message = emsg; + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + /* abort all pending requests */ + GNUNET_CLIENT_disconnect (dc->client); + dc->client = NULL; + return; + } + } if (sm->depth == dc->treedepth) { app = size; @@ -247,7 +393,37 @@ process_result (struct GNUNET_FS_DownloadContext *dc, } dc->completed += app; } - // FIXME: call progress callback + + pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; + make_download_status (&pi, dc); + pi.value.download.specifics.progress.data = pt; + pi.value.download.specifics.progress.offset = sm->offset; + pi.value.download.specifics.progress.data_len = size; + pi.value.download.specifics.progress.depth = sm->depth; + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + GNUNET_assert (dc->completed <= dc->length); + if (dc->completed == dc->length) + { + /* truncate file to size (since we store IBlocks at the end) */ + if (dc->handle != NULL) + { + GNUNET_DISK_file_close (dc->handle); + dc->handle = NULL; + if (0 != truncate (dc->filename, + GNUNET_ntohll (dc->uri->data.chk.file_length))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, + "truncate", + dc->filename); + } + /* signal completion */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; + make_download_status (&pi, dc); + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + GNUNET_assert (sm->depth == dc->treedepth); + } + // FIXME: make persistent if (sm->depth == dc->treedepth) return; GNUNET_assert (0 == (size % sizeof(struct ContentHashKey))); @@ -458,6 +634,7 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc) * * @param h handle to the file sharing subsystem * @param uri the URI of the file (determines what to download); CHK or LOC URI + * @param meta known metadata for the file (can be NULL) * @param filename where to store the file, maybe NULL (then no file is * created on disk and data must be grabbed from the callbacks) * @param offset at what offset should we start the download (typically 0) @@ -471,6 +648,7 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc) struct GNUNET_FS_DownloadContext * GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, uint64_t offset, uint64_t length, @@ -478,6 +656,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, enum GNUNET_FS_DownloadOptions options, struct GNUNET_FS_DownloadContext *parent) { + struct GNUNET_FS_ProgressInfo pi; struct GNUNET_FS_DownloadContext *dc; struct GNUNET_CLIENT_Connection *client; @@ -499,6 +678,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, dc->client = client; dc->parent = parent; dc->uri = GNUNET_FS_uri_dup (uri); + dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); if (NULL != filename) { dc->filename = GNUNET_strdup (filename); @@ -519,6 +699,7 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, _("Download failed: could not open file `%s': %s\n"), dc->filename, STRERROR (errno)); + GNUNET_CONTAINER_meta_data_destroy (dc->meta); GNUNET_FS_uri_destroy (dc->uri); GNUNET_free (dc->filename); GNUNET_CLIENT_disconnect (dc->client); @@ -547,7 +728,12 @@ GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, &receive_results, dc, GNUNET_TIME_UNIT_FOREVER_REL); - // FIXME: signal download start + pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; + make_download_status (&pi, dc); + pi.value.download.specifics.start.meta = meta; + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + return dc; } @@ -580,9 +766,14 @@ void GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete) { - // FIXME: make unpersistent - // FIXME: signal download end - + struct GNUNET_FS_ProgressInfo pi; + + // FIXME: make unpersistent + pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; + make_download_status (&pi, dc); + dc->client_info = dc->h->upcb (dc->h->upcb_cls, + &pi); + if (GNUNET_SCHEDULER_NO_TASK != dc->task) GNUNET_SCHEDULER_cancel (dc->h->sched, dc->task); @@ -594,7 +785,8 @@ GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc, GNUNET_CONTAINER_multihashmap_destroy (dc->active); if (dc->filename != NULL) { - GNUNET_DISK_file_close (dc->handle); + if (NULL != dc->handle) + GNUNET_DISK_file_close (dc->handle); if ( (dc->completed != dc->length) && (GNUNET_YES == do_delete) ) { @@ -605,75 +797,15 @@ GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc, } GNUNET_free (dc->filename); } + GNUNET_CONTAINER_meta_data_destroy (dc->meta); GNUNET_FS_uri_destroy (dc->uri); GNUNET_free (dc); } - - - - - - - - - - - - - - - #if 0 -/** - * Compute how many bytes of data are stored in - * this node. - */ -static unsigned int -get_node_size (const struct Node *node) -{ - unsigned int i; - unsigned int ret; - unsigned long long rsize; - unsigned long long spos; - unsigned long long epos; - - GNUNET_GE_ASSERT (node->ctx->ectx, node->offset < node->ctx->total); - if (node->level == 0) - { - ret = GNUNET_ECRS_DBLOCK_SIZE; - if (node->offset + (unsigned long long) ret > node->ctx->total) - ret = (unsigned int) (node->ctx->total - node->offset); -#if DEBUG_DOWNLOAD - GNUNET_GE_LOG (node->ctx->rm->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Node at offset %llu and level %d has size %u\n", - node->offset, node->level, ret); -#endif - return ret; - } - rsize = GNUNET_ECRS_DBLOCK_SIZE; - for (i = 0; i < node->level - 1; i++) - rsize *= GNUNET_ECRS_CHK_PER_INODE; - spos = rsize * (node->offset / sizeof (GNUNET_EC_ContentHashKey)); - epos = spos + rsize * GNUNET_ECRS_CHK_PER_INODE; - if (epos > node->ctx->total) - epos = node->ctx->total; - ret = (epos - spos) / rsize; - if (ret * rsize < epos - spos) - ret++; /* need to round up! */ -#if DEBUG_DOWNLOAD - GNUNET_GE_LOG (node->ctx->rm->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "Node at offset %llu and level %d has size %u\n", - node->offset, node->level, - ret * sizeof (GNUNET_EC_ContentHashKey)); -#endif - return ret * sizeof (GNUNET_EC_ContentHashKey); -} /** * Check if self block is already present on the drive. If the block diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index 5de9cbeb8..e173b1149 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -192,8 +192,8 @@ GNUNET_FS_file_information_create_from_file (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime) { struct FileInfo *fi; @@ -280,8 +280,8 @@ GNUNET_FS_file_information_create_from_data (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime) { return GNUNET_FS_file_information_create_from_reader (client_info, @@ -323,8 +323,8 @@ GNUNET_FS_file_information_create_from_reader (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime) { struct GNUNET_FS_FileInformation *ret; @@ -387,12 +387,12 @@ struct DirScanCls /** * Desired anonymity level. */ - unsigned int anonymity; + uint32_t anonymity; /** * Desired publishing priority. */ - unsigned int priority; + uint32_t priority; /** * Expiration time for publication. @@ -499,8 +499,8 @@ int GNUNET_FS_directory_scanner_default (void *cls, const char *dirname, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime, GNUNET_FS_FileProcessor proc, void *proc_cls, @@ -593,8 +593,8 @@ GNUNET_FS_file_information_create_from_directory (void *client_info, GNUNET_FS_DirectoryScanner scanner, void *scanner_cls, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime, char **emsg) { @@ -656,8 +656,8 @@ struct GNUNET_FS_FileInformation * GNUNET_FS_file_information_create_empty_directory (void *client_info, const struct GNUNET_CONTAINER_MetaData *meta, const struct GNUNET_FS_Uri *keywords, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime) { struct GNUNET_FS_FileInformation *ret; diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c index fb45d99e9..64763d164 100644 --- a/src/fs/fs_namespace.c +++ b/src/fs/fs_namespace.c @@ -46,9 +46,9 @@ struct GNUNET_FS_Uri * GNUNET_FS_namespace_advertise (struct GNUNET_FS_Handle *h, struct GNUNET_FS_Namespace *namespace, - const struct GNUNET_MetaData *meta, - unsigned int anonymity, - unsigned int priority, + const struct GNUNET_CONTAINER_MetaData *meta, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expiration, const struct GNUNET_FS_Uri *advertisementURI, const char *rootEntry) @@ -284,9 +284,9 @@ write_namespace_key (struct GNUNET_GC_Configuration *cfg, struct GNUNET_ECRS_URI * GNUNET_ECRS_namespace_create (struct GNUNET_GE_Context *ectx, struct GNUNET_GC_Configuration *cfg, - const struct GNUNET_MetaData *meta, - unsigned int anonymityLevel, - unsigned int priority, + const struct GNUNET_CONTAINER_MetaData *meta, + uint32_t anonymityLevel, + uint32_t priority, GNUNET_CronTime expiration, const struct GNUNET_ECRS_URI *advertisementURI, const char *rootEntry) @@ -519,8 +519,8 @@ struct GNUNET_ECRS_URI * GNUNET_ECRS_namespace_add_content (struct GNUNET_GE_Context *ectx, struct GNUNET_GC_Configuration *cfg, const GNUNET_HashCode * pid, - unsigned int anonymityLevel, - unsigned int priority, + uint32_t anonymityLevel, + uint32_t priority, GNUNET_CronTime expiration, const char *thisId, const char *nextId, diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 422984ecb..d58ea3f37 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -467,7 +467,7 @@ static void block_proc (void *cls, const GNUNET_HashCode *query, uint64_t offset, - unsigned int type, + uint32_t type, const void *block, uint16_t block_size) { @@ -909,8 +909,8 @@ fip_signal_start(void *cls, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, - unsigned int *anonymity, - unsigned int *priority, + uint32_t *anonymity, + uint32_t *priority, struct GNUNET_TIME_Absolute *expirationTime, void **client_info) { @@ -1021,8 +1021,8 @@ fip_signal_stop(void *cls, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, - unsigned int *anonymity, - unsigned int *priority, + uint32_t *anonymity, + uint32_t *priority, struct GNUNET_TIME_Absolute *expirationTime, void **client_info) { @@ -1135,12 +1135,12 @@ struct PublishKskContext /** * Anonymity level for the KBlocks. */ - unsigned int anonymity; + uint32_t anonymity; /** * Priority for the KBlocks. */ - unsigned int priority; + uint32_t priority; }; @@ -1284,8 +1284,8 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri *uri, struct GNUNET_TIME_Absolute expirationTime, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls) @@ -1443,8 +1443,8 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri *uri, struct GNUNET_TIME_Absolute expirationTime, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls) diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 109a76791..fd7dcf6d9 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c @@ -274,7 +274,7 @@ process_ksk_result (struct GNUNET_FS_SearchContext *sc, static struct GNUNET_FS_SearchContext * search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, - unsigned int anonymity, + uint32_t anonymity, struct GNUNET_FS_SearchContext *parent); @@ -738,7 +738,7 @@ try_reconnect (struct GNUNET_FS_SearchContext *sc) static struct GNUNET_FS_SearchContext * search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, - unsigned int anonymity, + uint32_t anonymity, struct GNUNET_FS_SearchContext *parent) { struct GNUNET_FS_SearchContext *sc; @@ -830,7 +830,7 @@ search_start (struct GNUNET_FS_Handle *h, struct GNUNET_FS_SearchContext * GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, - unsigned int anonymity) + uint32_t anonymity) { return search_start (h, uri, anonymity, NULL); } diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index 4186a4e21..bfbd7019b 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h @@ -67,7 +67,7 @@ struct GNUNET_FS_TreeEncoder; typedef void (*GNUNET_FS_TreeBlockProcessor)(void *cls, const GNUNET_HashCode *query, uint64_t offset, - unsigned int type, + uint32_t type, const void *block, uint16_t block_size); diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index 23779bc97..03dae383f 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -200,7 +200,7 @@ static void unindex_process (void *cls, const GNUNET_HashCode *query, uint64_t offset, - unsigned int type, + uint32_t type, const void *block, uint16_t block_size) { diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index fb6b7f133..ee10a0156 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -415,7 +415,7 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext* * the metadata must be passed as the "scls" argument. * * @param ctx command line processor context - * @param scls must be of type "struct GNUNET_MetaData **" + * @param scls must be of type "struct GNUNET_CONTAINER_MetaData **" * @param option name of the option (typically 'k') * @param value command line argument given * @return GNUNET_OK on success @@ -864,12 +864,11 @@ struct GNUNET_FS_ProgressInfo const struct GNUNET_FS_Uri *uri; /** - * How large is the file overall? For directories, - * this is only the size of the directory itself, - * not of the other files contained within the - * directory. + * How large is the download overall? This + * is NOT necessarily the size from the + * URI since we may be doing a partial download. */ - uint64_t size; + uint64_t length; /** * At what time do we expect to finish the download? @@ -938,8 +937,8 @@ struct GNUNET_FS_ProgressInfo /** * Known metadata for the download. */ - const struct GNUNET_MetaData *meta; - + const struct GNUNET_CONTAINER_MetaData *meta; + } start; /** @@ -951,7 +950,7 @@ struct GNUNET_FS_ProgressInfo /** * Known metadata for the download. */ - const struct GNUNET_MetaData *meta; + const struct GNUNET_CONTAINER_MetaData *meta; /** * Error message, NULL if we have not encountered any error yet. @@ -1449,8 +1448,8 @@ typedef int (*GNUNET_FS_FileInformationProcessor)(void *cls, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, - unsigned int *anonymity, - unsigned int *priority, + uint32_t *anonymity, + uint32_t *priority, struct GNUNET_TIME_Absolute *expirationTime, void **client_info); @@ -1513,8 +1512,8 @@ GNUNET_FS_file_information_create_from_file (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime); @@ -1543,8 +1542,8 @@ GNUNET_FS_file_information_create_from_data (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime); @@ -1597,8 +1596,8 @@ GNUNET_FS_file_information_create_from_reader (void *client_info, const struct GNUNET_FS_Uri *keywords, const struct GNUNET_CONTAINER_MetaData *meta, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime); @@ -1634,8 +1633,8 @@ typedef void (*GNUNET_FS_FileProcessor)(void *cls, typedef int (*GNUNET_FS_DirectoryScanner)(void *cls, const char *dirname, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime, GNUNET_FS_FileProcessor proc, void *proc_cls, @@ -1669,8 +1668,8 @@ int GNUNET_FS_directory_scanner_default (void *cls, const char *dirname, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime, GNUNET_FS_FileProcessor proc, void *proc_cls, @@ -1705,8 +1704,8 @@ GNUNET_FS_file_information_create_from_directory (void *client_info, GNUNET_FS_DirectoryScanner scanner, void *scanner_cls, int do_index, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime, char **emsg); @@ -1731,8 +1730,8 @@ struct GNUNET_FS_FileInformation * GNUNET_FS_file_information_create_empty_directory (void *client_info, const struct GNUNET_CONTAINER_MetaData *meta, const struct GNUNET_FS_Uri *keywords, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expirationTime); @@ -1874,8 +1873,8 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri *uri, struct GNUNET_TIME_Absolute expirationTime, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls); @@ -1904,8 +1903,8 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri *uri, struct GNUNET_TIME_Absolute expirationTime, - unsigned int anonymity, - unsigned int priority, + uint32_t anonymity, + uint32_t priority, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls); @@ -1983,9 +1982,9 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc); struct GNUNET_FS_Uri * GNUNET_FS_namespace_advertise (struct GNUNET_FS_Handle *h, struct GNUNET_FS_Namespace *namespace, - const struct GNUNET_MetaData *meta, - unsigned int anonymity, - unsigned int priority, + const struct GNUNET_CONTAINER_MetaData *meta, + uint32_t anonymity, + uint32_t priority, struct GNUNET_TIME_Absolute expiration, const struct GNUNET_FS_Uri *advertisementURI, const char *rootEntry); @@ -2092,7 +2091,7 @@ GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Namespace *namespace, struct GNUNET_FS_SearchContext * GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, - unsigned int anonymity); + uint32_t anonymity); /** @@ -2165,6 +2164,7 @@ enum GNUNET_FS_DownloadOptions * * @param h handle to the file sharing subsystem * @param uri the URI of the file (determines what to download); CHK or LOC URI + * @param meta known metadata for the file (can be NULL) * @param filename where to store the file, maybe NULL (then no file is * created on disk and data must be grabbed from the callbacks) * @param offset at what offset should we start the download (typically 0) @@ -2178,6 +2178,7 @@ enum GNUNET_FS_DownloadOptions struct GNUNET_FS_DownloadContext * GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, uint64_t offset, uint64_t length,