X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Ffs%2Ffs_publish.c;h=2bb7c575045c07421e66c1175adda870cdcc3d6e;hb=4e2504a967ba09643c6dd7e3b9ce400e30adcb3d;hp=be2994176998172f48fd98c8eb3c19b2915db487;hpb=bfa8316970a7dce659585a038458bc41776ed965;p=oweals%2Fgnunet.git diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index be2994176..2bb7c5750 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -1,129 +1,101 @@ /* This file is part of GNUnet. - (C) 2009, 2010 Christian Grothoff (and other contributing authors) + Copyright (C) 2009, 2010 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . */ - /** * @file fs/fs_publish.c * @brief publish a file or directory in GNUnet - * @see http://gnunet.org/encoding + * @see https://gnunet.org/encoding * @author Krista Bennett * @author Christian Grothoff - * - * TODO: - * - indexing cleanup: unindex on failure (can wait) - * - datastore reservation support (optimization) - * - location URIs (publish with anonymity-level zero) */ - #include "platform.h" #include "gnunet_constants.h" #include "gnunet_signatures.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" -#define DEBUG_PUBLISH GNUNET_NO - - -/** - * Context for "ds_put_cont". - */ -struct PutContCtx -{ - /** - * Current publishing context. - */ - struct GNUNET_FS_PublishContext *sc; - - /** - * Specific file with the block. - */ - struct GNUNET_FS_FileInformation *p; - - /** - * Function to run next, if any (can be NULL). - */ - GNUNET_SCHEDULER_Task cont; - - /** - * Closure for cont. - */ - void *cont_cls; -}; - /** - * Fill in all of the generic fields for + * Fill in all of the generic fields for * a publish event and call the callback. * * @param pi structure to fill in - * @param sc overall publishing context + * @param pc overall publishing context * @param p file information for the file being published * @param offset where in the file are we so far * @return value returned from callback */ void * GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_PublishContext *sc, - const struct GNUNET_FS_FileInformation *p, - uint64_t offset) + struct GNUNET_FS_PublishContext *pc, + const struct GNUNET_FS_FileInformation *p, + uint64_t offset) { - pi->value.publish.sc = sc; + pi->value.publish.pc = pc; pi->value.publish.fi = p; - pi->value.publish.cctx - = p->client_info; - pi->value.publish.pctx - = (NULL == p->dir) ? NULL : p->dir->client_info; + pi->value.publish.cctx = p->client_info; + pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info; pi->value.publish.filename = p->filename; - pi->value.publish.size - = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size; - pi->value.publish.eta - = GNUNET_TIME_calculate_eta (p->start_time, - offset, - pi->value.publish.size); + pi->value.publish.size = + (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size; + pi->value.publish.eta = + GNUNET_TIME_calculate_eta (p->start_time, offset, + pi->value.publish.size); pi->value.publish.completed = offset; - pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time); - pi->value.publish.anonymity = p->anonymity; - return sc->h->upcb (sc->h->upcb_cls, - pi); + pi->value.publish.duration = + GNUNET_TIME_absolute_get_duration (p->start_time); + pi->value.publish.anonymity = p->bo.anonymity_level; + pi->fsh = pc->h; + return pc->h->upcb (pc->h->upcb_cls, pi); } /** * Cleanup the publish context, we're done with it. * - * @param pc struct to clean up after + * @param pc struct to clean up */ static void publish_cleanup (struct GNUNET_FS_PublishContext *pc) { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Cleaning up publish context (done!)\n"); + if (NULL != pc->fhc) + { + GNUNET_CRYPTO_hash_file_cancel (pc->fhc); + pc->fhc = NULL; + } GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); - if (pc->namespace != NULL) - GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO); - GNUNET_free_non_null (pc->nid); + GNUNET_free_non_null (pc->nid); GNUNET_free_non_null (pc->nuid); GNUNET_free_non_null (pc->serialization); - if (pc->dsh != NULL) + if (NULL != pc->dsh) + { GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); - if (pc->client != NULL) - GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO); + pc->dsh = NULL; + } + if (NULL != pc->mq) + { + GNUNET_MQ_destroy (pc->mq); + pc->mq = NULL; + } + GNUNET_assert (NULL == pc->upload_task); GNUNET_free (pc); } @@ -132,45 +104,48 @@ publish_cleanup (struct GNUNET_FS_PublishContext *pc) * Function called by the datastore API with * the result from the PUT request. * - * @param cls our closure - * @param success GNUNET_OK on success + * @param cls the `struct GNUNET_FS_PublishContext *` + * @param success #GNUNET_OK on success + * @param min_expiration minimum expiration time required for content to be stored * @param msg error message (or NULL) */ static void ds_put_cont (void *cls, - int success, - const char *msg) + int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) { - struct PutContCtx *pcc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_ProgressInfo pi; - if (GNUNET_SYSERR == pcc->sc->in_network_wait) + pc->qre = NULL; + if (GNUNET_SYSERR == success) + { + GNUNET_asprintf (&pc->fi_pos->emsg, + _("Publishing failed: %s"), + msg); + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = pc->fi_pos->emsg; + pc->fi_pos->client_info = + GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0); + if ((GNUNET_YES != pc->fi_pos->is_directory) && + (NULL != pc->fi_pos->filename) && + (GNUNET_YES == pc->any_done) && + (GNUNET_YES == pc->fi_pos->data.file.do_index)) { - /* we were aborted in the meantime, - finish shutdown! */ - publish_cleanup (pcc->sc); - GNUNET_free (pcc); - return; + /* run unindex to clean up */ + GNUNET_FS_unindex_start (pc->h, + pc->fi_pos->filename, + NULL); } - GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait); - pcc->sc->in_network_wait = GNUNET_NO; - if (GNUNET_OK != success) - { - GNUNET_asprintf (&pcc->p->emsg, - _("Upload failed: %s"), - msg); - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = pcc->p->emsg; - pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0); - } - if (NULL != pcc->cont) - pcc->sc->upload_task - = GNUNET_SCHEDULER_add_with_priority (pcc->sc->h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - pcc->cont, - pcc->cont_cls); - GNUNET_free (pcc); + return; + } + pc->any_done = GNUNET_YES; + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); } @@ -180,19 +155,21 @@ ds_put_cont (void *cls, * published. * * @param p the completed upload - * @param sc context of the publication + * @param pc context of the publication */ -static void +static void signal_publish_completion (struct GNUNET_FS_FileInformation *p, - struct GNUNET_FS_PublishContext *sc) + struct GNUNET_FS_PublishContext *pc) { struct GNUNET_FS_ProgressInfo pi; - + pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED; pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO; pi.value.publish.specifics.completed.chk_uri = p->chk_uri; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, - GNUNET_ntohll (p->chk_uri->data.chk.file_length)); + pi.value.publish.specifics.completed.sks_uri = p->sks_uri; + p->client_info = + GNUNET_FS_publish_make_status_ (&pi, pc, p, + p->data.file.file_size); } @@ -202,21 +179,56 @@ signal_publish_completion (struct GNUNET_FS_FileInformation *p, * a problem during publication. * * @param p the upload that had trouble - * @param sc context of the publication + * @param pc context of the publication * @param emsg error message */ -static void +static void signal_publish_error (struct GNUNET_FS_FileInformation *p, - struct GNUNET_FS_PublishContext *sc, - const char *emsg) + struct GNUNET_FS_PublishContext *pc, + const char *emsg) { struct GNUNET_FS_ProgressInfo pi; - + p->emsg = GNUNET_strdup (emsg); pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message =emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0); + pi.value.publish.specifics.error.message = emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + if ((p->is_directory != GNUNET_YES) && + (NULL != p->filename) && + (GNUNET_YES == pc->any_done) && + (p->data.file.do_index == GNUNET_YES)) + { + /* run unindex to clean up */ + GNUNET_FS_unindex_start (pc->h, + p->filename, + NULL); + } + +} + + +/** + * Datastore returns from reservation cancel request. + * + * @param cls the `struct GNUNET_FS_PublishContext *` + * @param success success code (not used) + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message (typically NULL, not used) + */ +static void +finish_release_reserve (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + pc->qre = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Releasing reserve done!\n"); + signal_publish_completion (pc->fi, pc); + pc->all_done = GNUNET_YES; + GNUNET_FS_publish_sync_ (pc); } @@ -224,28 +236,40 @@ signal_publish_error (struct GNUNET_FS_FileInformation *p, * We've finished publishing the SBlock as part of a larger upload. * Check the result and complete the larger upload. * - * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload + * @param cls the `struct GNUNET_FS_PublishContext *` of the larger upload * @param uri URI of the published SBlock * @param emsg NULL on success, otherwise error message */ static void publish_sblocks_cont (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *emsg) + const struct GNUNET_FS_Uri *uri, + const char *emsg) { struct GNUNET_FS_PublishContext *pc = cls; + + pc->sks_pc = NULL; if (NULL != emsg) - { - signal_publish_error (pc->fi, - pc, - emsg); - GNUNET_FS_publish_sync_ (pc); - return; - } - // FIXME: release the datastore reserve here! - signal_publish_completion (pc->fi, pc); - pc->all_done = GNUNET_YES; - GNUNET_FS_publish_sync_ (pc); + { + signal_publish_error (pc->fi, pc, emsg); + GNUNET_FS_publish_sync_ (pc); + return; + } + if (NULL != uri) + { + /* sks publication, remember namespace URI */ + pc->fi->sks_uri = GNUNET_FS_uri_dup (uri); + } + GNUNET_assert (pc->qre == NULL); + if ((pc->dsh != NULL) && (pc->rid != 0)) + { + pc->qre = + GNUNET_DATASTORE_release_reserve (pc->dsh, pc->rid, UINT_MAX, UINT_MAX, + &finish_release_reserve, pc); + } + else + { + finish_release_reserve (pc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + } } @@ -253,26 +277,23 @@ publish_sblocks_cont (void *cls, * We are almost done publishing the structure, * add SBlocks (if needed). * - * @param sc overall upload data + * @param pc overall upload data */ static void -publish_sblock (struct GNUNET_FS_PublishContext *sc) +publish_sblock (struct GNUNET_FS_PublishContext *pc) { - if (NULL != sc->namespace) - GNUNET_FS_publish_sks (sc->h, - sc->namespace, - sc->nid, - sc->nuid, - sc->fi->meta, - sc->fi->chk_uri, - sc->fi->expirationTime, - sc->fi->anonymity, - sc->fi->priority, - sc->options, - &publish_sblocks_cont, - sc); + if (NULL != pc->ns) + pc->sks_pc = GNUNET_FS_publish_sks (pc->h, + pc->ns, + pc->nid, + pc->nuid, + pc->fi->meta, + pc->fi->chk_uri, + &pc->fi->bo, + pc->options, + &publish_sblocks_cont, pc); else - publish_sblocks_cont (sc, NULL, NULL); + publish_sblocks_cont (pc, NULL, NULL); } @@ -280,44 +301,50 @@ publish_sblock (struct GNUNET_FS_PublishContext *sc) * We've finished publishing a KBlock as part of a larger upload. * Check the result and continue the larger upload. * - * @param cls the "struct GNUNET_FS_PublishContext*" + * @param cls the `struct GNUNET_FS_PublishContext *` * of the larger upload * @param uri URI of the published blocks * @param emsg NULL on success, otherwise error message */ static void publish_kblocks_cont (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *emsg) + const struct GNUNET_FS_Uri *uri, + const char *emsg) { struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p = pc->fi_pos; + pc->ksk_pc = NULL; if (NULL != emsg) - { - signal_publish_error (p, pc, emsg); - GNUNET_FS_file_information_sync_ (p); - GNUNET_FS_publish_sync_ (pc); - pc->upload_task - = GNUNET_SCHEDULER_add_with_priority (pc->h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - pc); - return; - } + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Error uploading KSK blocks: %s\n", + emsg); + signal_publish_error (p, pc, emsg); + GNUNET_FS_file_information_sync_ (p); + GNUNET_FS_publish_sync_ (pc); + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, + pc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "KSK blocks published, moving on to next file\n"); if (NULL != p->dir) - signal_publish_completion (p, pc); + signal_publish_completion (p, pc); /* move on to next file */ if (NULL != p->next) pc->fi_pos = p->next; else pc->fi_pos = p->dir; GNUNET_FS_publish_sync_ (pc); - pc->upload_task - = GNUNET_SCHEDULER_add_with_priority (pc->h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - pc); + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); } @@ -336,41 +363,43 @@ publish_kblocks_cont (void *cls, */ static size_t block_reader (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) + uint64_t offset, + size_t max, + void *buf, + char **emsg) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p; - size_t pt_size; const char *dd; + size_t pt_size; - p = sc->fi_pos; - if (p->is_directory) - { - pt_size = GNUNET_MIN(max, - p->data.dir.dir_size - offset); - dd = p->data.dir.dir_data; - memcpy (buf, - &dd[offset], - pt_size); - } + p = pc->fi_pos; + if (GNUNET_YES == p->is_directory) + { + pt_size = GNUNET_MIN (max, p->data.dir.dir_size - offset); + dd = p->data.dir.dir_data; + GNUNET_memcpy (buf, &dd[offset], pt_size); + } else + { + if (UINT64_MAX == offset) { - pt_size = GNUNET_MIN(max, - p->data.file.file_size - offset); - if (pt_size == 0) - return 0; /* calling reader with pt_size==0 - might free buf, so don't! */ - if (pt_size != - p->data.file.reader (p->data.file.reader_cls, - offset, - pt_size, - buf, - emsg)) - return 0; + if (&GNUNET_FS_data_reader_file_ == p->data.file.reader) + { + /* force closing the file to avoid keeping too many files open */ + p->data.file.reader (p->data.file.reader_cls, offset, 0, NULL, NULL); + } + return 0; } + pt_size = GNUNET_MIN (max, p->data.file.file_size - offset); + if (0 == pt_size) + return 0; /* calling reader with pt_size==0 + * might free buf, so don't! */ + if (pt_size != + p->data.file.reader (p->data.file.reader_cls, offset, pt_size, buf, + emsg)) + return 0; + } return pt_size; } @@ -381,132 +410,142 @@ block_reader (void *cls, * the final result. * * @param cls our publishing context - * @param tc scheduler's task context (not used) */ -static void -encode_cont (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) +static void +encode_cont (void *cls) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p; struct GNUNET_FS_ProgressInfo pi; char *emsg; - - p = sc->fi_pos; - GNUNET_FS_tree_encoder_finish (p->te, - &p->chk_uri, - &emsg); + uint64_t flen; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Finished with tree encoder\n"); + p = pc->fi_pos; + p->chk_uri = GNUNET_FS_tree_encoder_get_uri (p->te); + GNUNET_FS_file_information_sync_ (p); + GNUNET_FS_tree_encoder_finish (p->te, &emsg); p->te = NULL; if (NULL != emsg) - { - GNUNET_asprintf (&p->emsg, - _("Upload failed: %s"), - emsg); - GNUNET_free (emsg); - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = p->emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0); - } - /* continue with main */ - sc->upload_task - = GNUNET_SCHEDULER_add_with_priority (sc->h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - sc); + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Error during tree walk: %s\n", + emsg); + GNUNET_asprintf (&p->emsg, + _("Publishing failed: %s"), + emsg); + GNUNET_free (emsg); + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = p->emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + } + else + { + /* final progress event */ + GNUNET_assert (NULL != p->chk_uri); + flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri); + pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; + pi.value.publish.specifics.progress.data = NULL; + pi.value.publish.specifics.progress.offset = flen; + pi.value.publish.specifics.progress.data_len = 0; + pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen); + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen); + } + /* continue with main */ /* continue with main */ + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); } /** * Function called asking for the current (encoded) * block to be processed. After processing the - * client should either call "GNUNET_FS_tree_encode_next" - * or (on error) "GNUNET_FS_tree_encode_finish". + * client should either call #GNUNET_FS_tree_encoder_next + * or (on error) #GNUNET_FS_tree_encoder_finish. * * @param cls closure - * @param query the query for the block (key for lookup in the datastore) + * @param chk content hash key for the block * @param offset offset of the block in the file + * @param depth depth of the block in the file, 0 for DBLOCK * @param type type of the block (IBLOCK or DBLOCK) * @param block the (encrypted) block - * @param block_size size of block (in bytes) + * @param block_size size of @a block (in bytes) */ -static void +static void block_proc (void *cls, - const GNUNET_HashCode *query, - uint64_t offset, - enum GNUNET_BLOCK_Type type, - const void *block, - uint16_t block_size) + const struct ContentHashKey *chk, + uint64_t offset, + unsigned int depth, + enum GNUNET_BLOCK_Type type, + const void *block, + uint16_t block_size) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p; - struct PutContCtx * dpc_cls; struct OnDemandBlock odb; - p = sc->fi_pos; - if (NULL == sc->dsh) - { - sc->upload_task - = GNUNET_SCHEDULER_add_with_priority (sc->h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - sc); - return; - } - - GNUNET_assert (GNUNET_NO == sc->in_network_wait); - sc->in_network_wait = GNUNET_YES; - dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx)); - dpc_cls->cont = &GNUNET_FS_publish_main_; - dpc_cls->cont_cls = sc; - dpc_cls->sc = sc; - dpc_cls->p = p; - if ( (! p->is_directory) && + p = pc->fi_pos; + if (NULL == pc->dsh) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Waiting for datastore connection\n"); + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); + return; + } + + if ( (GNUNET_YES != p->is_directory) && (GNUNET_YES == p->data.file.do_index) && - (type == GNUNET_BLOCK_TYPE_DBLOCK) ) - { -#if DEBUG_PUBLISH - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Indexing block `%s' for offset %llu with index size %u\n", - GNUNET_h2s (query), - (unsigned long long) offset, - sizeof (struct OnDemandBlock)); -#endif - odb.offset = GNUNET_htonll (offset); - odb.file_id = p->data.file.file_id; - GNUNET_DATASTORE_put (sc->dsh, - sc->rid, - query, - sizeof(struct OnDemandBlock), - &odb, - GNUNET_BLOCK_TYPE_ONDEMAND, - p->priority, - p->anonymity, - p->expirationTime, - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &ds_put_cont, - dpc_cls); - return; - } -#if DEBUG_PUBLISH + (GNUNET_BLOCK_TYPE_FS_DBLOCK == type) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Indexing block `%s' for offset %llu with index size %u\n", + GNUNET_h2s (&chk->query), + (unsigned long long) offset, + (unsigned int) sizeof (struct OnDemandBlock)); + odb.offset = GNUNET_htonll (offset); + odb.file_id = p->data.file.file_id; + GNUNET_assert (pc->qre == NULL); + pc->qre = + GNUNET_DATASTORE_put (pc->dsh, + (p->is_directory == GNUNET_YES) ? 0 : pc->rid, + &chk->query, + sizeof (struct OnDemandBlock), + &odb, + GNUNET_BLOCK_TYPE_FS_ONDEMAND, + p->bo.content_priority, + p->bo.anonymity_level, + p->bo.replication_level, + p->bo.expiration_time, + -2, 1, + &ds_put_cont, pc); + return; + } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing block `%s' for offset %llu with size %u\n", - GNUNET_h2s (query), - (unsigned long long) offset, - (unsigned int) block_size); -#endif - GNUNET_DATASTORE_put (sc->dsh, - sc->rid, - query, - block_size, - block, - type, - p->priority, - p->anonymity, - p->expirationTime, - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &ds_put_cont, - dpc_cls); + "Publishing block `%s' for offset %llu with size %u\n", + GNUNET_h2s (&chk->query), + (unsigned long long) offset, + (unsigned int) block_size); + GNUNET_assert (pc->qre == NULL); + pc->qre = + GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : pc->rid, + &chk->query, + block_size, + block, + type, + p->bo.content_priority, + p->bo.anonymity_level, + p->bo.replication_level, + p->bo.expiration_time, + -2, 1, + &ds_put_cont, + pc); } @@ -517,27 +556,44 @@ block_proc (void *cls, * @param cls closure * @param offset where are we in the file * @param pt_block plaintext of the currently processed block - * @param pt_size size of pt_block - * @param depth depth of the block in the tree + * @param pt_size size of @a pt_block + * @param depth depth of the block in the tree, 0 for DBLOCK */ -static void -progress_proc (void *cls, - uint64_t offset, - const void *pt_block, - size_t pt_size, - unsigned int depth) -{ - struct GNUNET_FS_PublishContext *sc = cls; +static void +progress_proc (void *cls, uint64_t offset, + const void *pt_block, + size_t pt_size, + unsigned int depth) +{ + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p; + struct GNUNET_FS_FileInformation *par; struct GNUNET_FS_ProgressInfo pi; - p = sc->fi_pos; + p = pc->fi_pos; pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; pi.value.publish.specifics.progress.data = pt_block; pi.value.publish.specifics.progress.offset = offset; pi.value.publish.specifics.progress.data_len = pt_size; pi.value.publish.specifics.progress.depth = depth; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset); + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset); + if ( (0 != depth) || + (GNUNET_YES == p->is_directory) ) + return; + while (NULL != (par = p->dir)) + { + p = par; + GNUNET_assert (GNUNET_YES == par->is_directory); + p->data.dir.contents_completed += pt_size; + pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY; + pi.value.publish.specifics.progress_directory.completed = p->data.dir.contents_completed; + pi.value.publish.specifics.progress_directory.total = p->data.dir.contents_size; + pi.value.publish.specifics.progress_directory.eta = GNUNET_TIME_calculate_eta (p->start_time, + p->data.dir.contents_completed, + p->data.dir.contents_size); + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + + } } @@ -546,10 +602,10 @@ progress_proc (void *cls, * block into memory, encrypt it and send it to the FS service. Then * continue with the main task. * - * @param sc overall upload data + * @param pc overall upload data */ static void -publish_content (struct GNUNET_FS_PublishContext *sc) +publish_content (struct GNUNET_FS_PublishContext *pc) { struct GNUNET_FS_FileInformation *p; char *emsg; @@ -558,120 +614,171 @@ publish_content (struct GNUNET_FS_PublishContext *sc) void *raw_data; uint64_t size; - p = sc->fi_pos; - GNUNET_assert (p != NULL); + p = pc->fi_pos; + GNUNET_assert (NULL != p); if (NULL == p->te) + { + if (GNUNET_YES == p->is_directory) { - if (p->is_directory) - { - db = GNUNET_FS_directory_builder_create (p->meta); - dirpos = p->data.dir.entries; - while (NULL != dirpos) - { - if (dirpos->is_directory) - { - raw_data = dirpos->data.dir.dir_data; - dirpos->data.dir.dir_data = NULL; - } - else - { - raw_data = NULL; - if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) && - (dirpos->data.file.file_size > 0) ) - { - raw_data = GNUNET_malloc (dirpos->data.file.file_size); - emsg = NULL; - if (dirpos->data.file.file_size != - dirpos->data.file.reader (dirpos->data.file.reader_cls, - 0, - dirpos->data.file.file_size, - raw_data, - &emsg)) - { - GNUNET_free_non_null (emsg); - GNUNET_free (raw_data); - raw_data = NULL; - } - } - } - GNUNET_FS_directory_builder_add (db, - dirpos->chk_uri, - dirpos->meta, - raw_data); - GNUNET_free_non_null (raw_data); - dirpos = dirpos->next; - } - GNUNET_FS_directory_builder_finish (db, - &p->data.dir.dir_size, - &p->data.dir.dir_data); - GNUNET_FS_file_information_sync_ (p); - } - size = (p->is_directory) - ? p->data.dir.dir_size - : p->data.file.file_size; - p->te = GNUNET_FS_tree_encoder_create (sc->h, - size, - sc, - &block_reader, - &block_proc, - &progress_proc, - &encode_cont); - + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n"); + db = GNUNET_FS_directory_builder_create (p->meta); + dirpos = p->data.dir.entries; + while (NULL != dirpos) + { + if (GNUNET_YES == dirpos->is_directory) + { + raw_data = dirpos->data.dir.dir_data; + dirpos->data.dir.dir_data = NULL; + } + else + { + raw_data = NULL; + if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) && + (dirpos->data.file.file_size > 0)) + { + raw_data = GNUNET_malloc (dirpos->data.file.file_size); + emsg = NULL; + if (dirpos->data.file.file_size != + dirpos->data.file.reader (dirpos->data.file.reader_cls, 0, + dirpos->data.file.file_size, raw_data, + &emsg)) + { + GNUNET_free_non_null (emsg); + GNUNET_free (raw_data); + raw_data = NULL; + } + dirpos->data.file.reader (dirpos->data.file.reader_cls, UINT64_MAX, 0, 0, NULL); + } + } + GNUNET_FS_directory_builder_add (db, dirpos->chk_uri, dirpos->meta, + raw_data); + GNUNET_free_non_null (raw_data); + dirpos = dirpos->next; + } + GNUNET_free_non_null (p->data.dir.dir_data); + p->data.dir.dir_data = NULL; + p->data.dir.dir_size = 0; + GNUNET_FS_directory_builder_finish (db, &p->data.dir.dir_size, + &p->data.dir.dir_data); + GNUNET_FS_file_information_sync_ (p); } + size = (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Creating tree encoder\n"); + p->te = + GNUNET_FS_tree_encoder_create (pc->h, size, pc, &block_reader, + &block_proc, &progress_proc, + &encode_cont); + + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing next block from tree\n"); GNUNET_FS_tree_encoder_next (p->te); } /** - * Process the response (or lack thereof) from - * the "fs" service to our 'start index' request. + * Check the response from the "fs" service to our 'start index' + * request. + * + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) + * @param msg the response we got + */ +static int +check_index_start_failed (void *cls, + const struct GNUNET_MessageHeader *msg) +{ + size_t msize = ntohs (msg->size) - sizeof (*msg); + const char *emsg = (const char *) &msg[1]; + + if (emsg[msize - sizeof (struct GNUNET_MessageHeader) - 1] != '\0') + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Process the response from the "fs" service to our 'start index' + * request. * - * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_) + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) * @param msg the response we got */ static void -process_index_start_response (void *cls, - const struct GNUNET_MessageHeader *msg) +handle_index_start_failed (void *cls, + const struct GNUNET_MessageHeader *msg) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_FileInformation *p; - const char *emsg; - uint16_t msize; + const char *emsg = (const char *) &msg[1]; - GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO); - sc->client = NULL; - p = sc->fi_pos; - if (msg == NULL) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _("timeout on index-start request to `fs' service")); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (sc); - return; - } - if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK) - { - msize = ntohs (msg->size); - emsg = (const char *) &msg[1]; - if ( (msize <= sizeof (struct GNUNET_MessageHeader)) || - (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') ) - emsg = gettext_noop ("unknown error"); - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - gettext (emsg)); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (sc); - return; - } + GNUNET_MQ_destroy (pc->mq); + pc->mq = NULL; + p = pc->fi_pos; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, + gettext (emsg)); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); +} + + +/** + * Process the response from the "fs" service to our 'start index' + * request. + * + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) + * @param msg the response we got + */ +static void +handle_index_start_ok (void *cls, + const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + + GNUNET_MQ_destroy (pc->mq); + pc->mq = NULL; + p = pc->fi_pos; p->data.file.index_start_confirmed = GNUNET_YES; - /* success! continue with indexing */ GNUNET_FS_file_information_sync_ (p); - publish_content (sc); + publish_content (pc); +} + + +/** + * Generic error handler, called with the appropriate error code and + * the same closure specified at the creation of the message queue. + * Not every message queue implementation supports an error handler. + * + * @param cls closure with the `struct GNUNET_FS_PublishContext *` + * @param error error code + */ +static void +index_mq_error_handler (void *cls, + enum GNUNET_MQ_Error error) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + + if (NULL != pc->mq) + { + GNUNET_MQ_destroy (pc->mq); + pc->mq = NULL; + } + p = pc->fi_pos; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, + _("error on index-start request to `fs' service")); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); } @@ -682,238 +789,372 @@ process_index_start_response (void *cls, * @param cls closure, our publishing context * @param res resulting hash, NULL on error */ -static void +static void hash_for_index_cb (void *cls, - const GNUNET_HashCode * - res) + const struct GNUNET_HashCode *res) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_MQ_MessageHandler handlers[] = { + GNUNET_MQ_hd_fixed_size (index_start_ok, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK, + struct GNUNET_MessageHeader, + pc), + GNUNET_MQ_hd_var_size (index_start_failed, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED, + struct GNUNET_MessageHeader, + pc), + GNUNET_MQ_handler_end () + }; struct GNUNET_FS_FileInformation *p; + struct GNUNET_MQ_Envelope *env; struct IndexStartMessage *ism; size_t slen; - struct GNUNET_CLIENT_Connection *client; - uint32_t dev; + uint64_t dev; uint64_t ino; char *fn; - p = sc->fi_pos; - if (NULL == res) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _("failed to compute hash")); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (sc); - return; - } + pc->fhc = NULL; + p = pc->fi_pos; + if (NULL == res) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, + _("failed to compute hash")); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } if (GNUNET_YES == p->data.file.index_start_confirmed) - { - publish_content (sc); - return; - } + { + publish_content (pc); + return; + } fn = GNUNET_STRINGS_filename_expand (p->filename); + GNUNET_assert (fn != NULL); slen = strlen (fn) + 1; - if (slen > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - fn, - _("filename too long")); - GNUNET_free (fn); - p->data.file.do_index = GNUNET_NO; - GNUNET_FS_file_information_sync_ (p); - publish_content (sc); - return; - } -#if DEBUG_PUBLISH + if (slen >= + GNUNET_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + fn, _("filename too long")); + GNUNET_free (fn); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Hash of indexed file `%s' is `%s'\n", - p->data.file.filename, - GNUNET_h2s (res)); -#endif - client = GNUNET_CLIENT_connect (sc->h->sched, - "fs", - sc->h->cfg); - if (NULL == client) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - p->filename, - _("could not connect to `fs' service")); - p->data.file.do_index = GNUNET_NO; - publish_content (sc); - GNUNET_free (fn); - return; - } + "Hash of indexed file `%s' is `%s'\n", + p->filename, + GNUNET_h2s (res)); + if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) + { + p->data.file.file_id = *res; + p->data.file.have_hash = GNUNET_YES; + p->data.file.index_start_confirmed = GNUNET_YES; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + GNUNET_free (fn); + return; + } + pc->mq = GNUNET_CLIENT_connect (pc->h->cfg, + "fs", + handlers, + &index_mq_error_handler, + pc); + if (NULL == pc->mq) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, + _("could not connect to `fs' service")); + p->data.file.do_index = GNUNET_NO; + publish_content (pc); + GNUNET_free (fn); + return; + } if (p->data.file.have_hash != GNUNET_YES) - { - p->data.file.file_id = *res; - p->data.file.have_hash = GNUNET_YES; - GNUNET_FS_file_information_sync_ (p); - } - ism = GNUNET_malloc (sizeof(struct IndexStartMessage) + - slen); - ism->header.size = htons(sizeof(struct IndexStartMessage) + - slen); - ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START); + { + p->data.file.file_id = *res; + p->data.file.have_hash = GNUNET_YES; + GNUNET_FS_file_information_sync_ (p); + } + env = GNUNET_MQ_msg_extra (ism, + slen, + GNUNET_MESSAGE_TYPE_FS_INDEX_START); if (GNUNET_OK == GNUNET_DISK_file_get_identifiers (p->filename, - &dev, - &ino)) - { - ism->device = htonl (dev); - ism->inode = GNUNET_htonll(ino); - } + &dev, + &ino)) + { + ism->device = GNUNET_htonll (dev); + ism->inode = GNUNET_htonll (ino); + } else - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to get file identifiers for `%s'\n"), - p->filename); - } + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + _("Failed to get file identifiers for `%s'\n"), + p->filename); + } ism->file_id = *res; - memcpy (&ism[1], - fn, - slen); + GNUNET_memcpy (&ism[1], + fn, + slen); GNUNET_free (fn); - sc->client = client; - GNUNET_break (GNUNET_YES == - GNUNET_CLIENT_transmit_and_get_response (client, - &ism->header, - GNUNET_TIME_UNIT_FOREVER_REL, - GNUNET_YES, - &process_index_start_response, - sc)); - GNUNET_free (ism); + GNUNET_MQ_send (pc->mq, + env); +} + + +/** + * We've computed the CHK/LOC URI, now publish the KSKs (if applicable). + * + * @param pc publishing context to do this for + */ +static void +publish_kblocks (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_FS_FileInformation *p; + + p = pc->fi_pos; + /* upload of "p" complete, publish KBlocks! */ + if (NULL != p->keywords) + { + pc->ksk_pc = GNUNET_FS_publish_ksk (pc->h, + p->keywords, + p->meta, + p->chk_uri, + &p->bo, + pc->options, + &publish_kblocks_cont, + pc); + } + else + { + publish_kblocks_cont (pc, p->chk_uri, NULL); + } +} + + +/** + * Process the response from the "fs" service to our LOC sign request. + * + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) + * @param sig the response we got + */ +static void +handle_signature_response (void *cls, + const struct ResponseLocSignatureMessage *sig) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + + p = pc->fi_pos; + p->chk_uri->type = GNUNET_FS_URI_LOC; + /* p->data.loc.fi kept from CHK before */ + p->chk_uri->data.loc.peer = sig->peer; + p->chk_uri->data.loc.expirationTime + = GNUNET_TIME_absolute_ntoh (sig->expiration_time); + p->chk_uri->data.loc.contentSignature = sig->signature; + GNUNET_FS_file_information_sync_ (p); + GNUNET_FS_publish_sync_ (pc); + publish_kblocks (pc); +} + + +/** + * Generic error handler, called with the appropriate error code and + * the same closure specified at the creation of the message queue. + * Not every message queue implementation supports an error handler. + * + * @param cls closure with the `struct GNUNET_FS_PublishContext *` + * @param error error code + */ +static void +loc_mq_error_handler (void *cls, + enum GNUNET_MQ_Error error) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + if (NULL != pc->mq) + { + GNUNET_MQ_destroy (pc->mq); + pc->mq = NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not create LOC URI. Will continue with CHK instead.\n")); + publish_kblocks (pc); +} + + +/** + * We're publishing without anonymity. Contact the FS service + * to create a signed LOC URI for further processing, then + * continue with KSKs. + * + * @param pc the publishing context do to this for + */ +static void +create_loc_uri (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_MQ_MessageHandler handlers[] = { + GNUNET_MQ_hd_fixed_size (signature_response, + GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE, + struct ResponseLocSignatureMessage, + pc), + GNUNET_MQ_handler_end () + }; + struct GNUNET_MQ_Envelope *env; + struct RequestLocSignatureMessage *req; + struct GNUNET_FS_FileInformation *p; + + if (NULL != pc->mq) + GNUNET_MQ_destroy (pc->mq); + pc->mq = GNUNET_CLIENT_connect (pc->h->cfg, + "fs", + handlers, + &loc_mq_error_handler, + pc); + if (NULL == pc->mq) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not create LOC URI. Will continue with CHK instead.\n")); + publish_kblocks (pc); + return; + } + p = pc->fi_pos; + env = GNUNET_MQ_msg (req, + GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN); + req->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); + req->expiration_time = GNUNET_TIME_absolute_hton (p->bo.expiration_time); + req->chk = p->chk_uri->data.chk.chk; + req->file_length = GNUNET_htonll (p->chk_uri->data.chk.file_length); + GNUNET_MQ_send (pc->mq, + env); } /** * Main function that performs the upload. * - * @param cls "struct GNUNET_FS_PublishContext" identifies the upload - * @param tc task context + * @param cls `struct GNUNET_FS_PublishContext *` identifies the upload */ void -GNUNET_FS_publish_main_ (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) +GNUNET_FS_publish_main_ (void *cls) { struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_ProgressInfo pi; struct GNUNET_FS_FileInformation *p; char *fn; - pc->upload_task = GNUNET_SCHEDULER_NO_TASK; + pc->upload_task = NULL; p = pc->fi_pos; if (NULL == p) - { - /* upload of entire hierarchy complete, - publish namespace entries */ - GNUNET_FS_publish_sync_ (pc); - publish_sblock (pc); - return; - } + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Publishing complete, now publishing SKS and KSK blocks.\n"); + /* upload of entire hierarchy complete, + * publish namespace entries */ + GNUNET_FS_publish_sync_ (pc); + publish_sblock (pc); + return; + } /* find starting position */ - while ( (p->is_directory) && + while ( (GNUNET_YES == p->is_directory) && (NULL != p->data.dir.entries) && (NULL == p->emsg) && (NULL == p->data.dir.entries->chk_uri) ) - { - p = p->data.dir.entries; - pc->fi_pos = p; - GNUNET_FS_publish_sync_ (pc); - } + { + p = p->data.dir.entries; + pc->fi_pos = p; + GNUNET_FS_publish_sync_ (pc); + } /* abort on error */ if (NULL != p->emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Error uploading: %s\n", + p->emsg); + /* error with current file, abort all + * related files as well! */ + while (NULL != p->dir) { - /* error with current file, abort all - related files as well! */ - while (NULL != p->dir) - { - fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, - EXTRACTOR_METATYPE_FILENAME); - p = p->dir; - if (fn != NULL) - { - GNUNET_asprintf (&p->emsg, - _("Recursive upload failed at `%s': %s"), - fn, - p->emsg); - GNUNET_free (fn); - } - else - { - GNUNET_asprintf (&p->emsg, - _("Recursive upload failed: %s"), - p->emsg); - } - pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; - pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; - pi.value.publish.specifics.error.message = p->emsg; - p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); - } - pc->all_done = GNUNET_YES; - GNUNET_FS_publish_sync_ (pc); - return; + fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + p = p->dir; + if (fn != NULL) + { + GNUNET_asprintf (&p->emsg, + _("Recursive upload failed at `%s': %s"), + fn, + p->emsg); + GNUNET_free (fn); + } + else + { + GNUNET_asprintf (&p->emsg, + _("Recursive upload failed: %s"), + p->emsg); + } + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = p->emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); } + pc->all_done = GNUNET_YES; + GNUNET_FS_publish_sync_ (pc); + return; + } /* handle completion */ if (NULL != p->chk_uri) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "File upload complete, now publishing KSK blocks.\n"); + GNUNET_FS_publish_sync_ (pc); + + if ( (0 == p->bo.anonymity_level) && + (GNUNET_YES != + GNUNET_FS_uri_test_loc (p->chk_uri)) ) { - GNUNET_FS_publish_sync_ (pc); - /* upload of "p" complete, publish KBlocks! */ - if (p->keywords != NULL) - { - GNUNET_FS_publish_ksk (pc->h, - p->keywords, - p->meta, - p->chk_uri, - p->expirationTime, - p->anonymity, - p->priority, - pc->options, - &publish_kblocks_cont, - pc); - } - else - { - publish_kblocks_cont (pc, - p->chk_uri, - NULL); - } - return; + /* zero anonymity, box CHK URI in LOC URI */ + create_loc_uri (pc); } - if ( (!p->is_directory) && - (p->data.file.do_index) ) + else { - if (NULL == p->filename) - { - p->data.file.do_index = GNUNET_NO; - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Can not index file `%s': %s. Will try to insert instead.\n"), - "", - _("needs to be an actual file")); - GNUNET_FS_file_information_sync_ (p); - publish_content (pc); - return; - } - if (p->data.file.have_hash) - { - hash_for_index_cb (pc, - &p->data.file.file_id); - } - else - { - p->start_time = GNUNET_TIME_absolute_get (); - GNUNET_CRYPTO_hash_file (pc->h->sched, - GNUNET_SCHEDULER_PRIORITY_IDLE, - p->filename, - HASHING_BLOCKSIZE, - &hash_for_index_cb, - pc); - } + publish_kblocks (pc); + } + return; + } + if ((GNUNET_YES != p->is_directory) && (p->data.file.do_index)) + { + if (NULL == p->filename) + { + p->data.file.do_index = GNUNET_NO; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Can not index file `%s': %s. Will try to insert instead.\n"), + "", + _("needs to be an actual file")); + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); return; } + if (p->data.file.have_hash) + { + hash_for_index_cb (pc, &p->data.file.file_id); + } + else + { + p->start_time = GNUNET_TIME_absolute_get (); + pc->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, p->filename, + HASHING_BLOCKSIZE, &hash_for_index_cb, pc); + } + return; + } publish_content (pc); } @@ -922,73 +1163,165 @@ GNUNET_FS_publish_main_ (void *cls, * Signal the FS's progress function that we are starting * an upload. * - * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) * @param fi the entry in the publish-structure * @param length length of the file or directory * @param meta metadata for the file or directory (can be modified) * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param anonymity pointer to selected anonymity level (can be modified) - * @param priority pointer to selected priority (can be modified) - * @param expirationTime pointer to selected expiration time (can be modified) + * @param bo block options + * @param do_index should we index? * @param client_info pointer to client context set upon creation (can be modified) - * @return GNUNET_OK to continue (always) + * @return #GNUNET_OK to continue (always) */ static int -fip_signal_start(void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_CONTAINER_MetaData *meta, - struct GNUNET_FS_Uri **uri, - uint32_t *anonymity, - uint32_t *priority, - struct GNUNET_TIME_Absolute *expirationTime, - void **client_info) +fip_signal_start (void *cls, + struct GNUNET_FS_FileInformation *fi, + uint64_t length, + struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, + int *do_index, + void **client_info) { - struct GNUNET_FS_PublishContext *sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_ProgressInfo pi; - + unsigned int kc; + uint64_t left; + + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting publish operation\n"); + if (*do_index) + { + /* space for on-demand blocks */ + pc->reserve_space += + ((length + DBLOCK_SIZE - + 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock); + } + else + { + /* space for DBlocks */ + pc->reserve_space += length; + } + /* entries for IBlocks and DBlocks, space for IBlocks */ + left = length; + while (1) + { + left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE; + pc->reserve_entries += left; + if (left <= 1) + break; + left = left * sizeof (struct ContentHashKey); + pc->reserve_space += left; + } + pc->reserve_entries++; + /* entries and space for keywords */ + if (NULL != *uri) + { + kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri); + pc->reserve_entries += kc; + pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc; + } pi.status = GNUNET_FS_STATUS_PUBLISH_START; - *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0); + *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0); GNUNET_FS_file_information_sync_ (fi); + if ((fi->is_directory) && (fi->dir != NULL)) + { + /* We are a directory, and we are not top-level; process entries in directory */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_start, pc); + } return GNUNET_OK; } /** - * Signal the FS's progress function that we are suspending + * Actually signal the FS's progress function that we are suspending * an upload. * - * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param pc the publish context of which a file is being suspended + */ +static void +suspend_operation (struct GNUNET_FS_FileInformation *fi, + struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + + if (NULL != pc->ksk_pc) + { + GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); + pc->ksk_pc = NULL; + } + if (NULL != pc->sks_pc) + { + GNUNET_FS_publish_sks_cancel (pc->sks_pc); + pc->sks_pc = NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Suspending publish operation\n"); + GNUNET_free_non_null (fi->serialization); + fi->serialization = NULL; + off = (NULL == fi->chk_uri) ? 0 : (GNUNET_YES == fi->is_directory) ? fi->data.dir.dir_size : fi->data.file.file_size; + pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND; + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); + if (NULL != pc->qre) + { + GNUNET_DATASTORE_cancel (pc->qre); + pc->qre = NULL; + } + if (NULL != pc->dsh) + { + GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); + pc->dsh = NULL; + } + pc->rid = 0; +} + + +/** + * Signal the FS's progress function that we are suspending + * an upload. Performs the recursion. + * + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) * @param fi the entry in the publish-structure * @param length length of the file or directory * @param meta metadata for the file or directory (can be modified) * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param anonymity pointer to selected anonymity level (can be modified) - * @param priority pointer to selected priority (can be modified) - * @param expirationTime pointer to selected expiration time (can be modified) + * @param bo block options + * @param do_index should we index? * @param client_info pointer to client context set upon creation (can be modified) - * @return GNUNET_OK to continue (always) + * @return #GNUNET_OK to continue (always) */ static int -fip_signal_suspend(void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_CONTAINER_MetaData *meta, - struct GNUNET_FS_Uri **uri, - uint32_t *anonymity, - uint32_t *priority, - struct GNUNET_TIME_Absolute *expirationTime, - void **client_info) +fip_signal_suspend (void *cls, + struct GNUNET_FS_FileInformation *fi, + uint64_t length, + struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, + int *do_index, + void **client_info) { - struct GNUNET_FS_PublishContext*sc = cls; - struct GNUNET_FS_ProgressInfo pi; - uint64_t off; + struct GNUNET_FS_PublishContext *pc = cls; - GNUNET_free_non_null (fi->serialization); - fi->serialization = NULL; - off = (fi->chk_uri == NULL) ? 0 : length; - pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND; - GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off)); + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) + { + /* process entries in directory */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_suspend, pc); + } + suspend_operation (fi, pc); *client_info = NULL; return GNUNET_OK; } @@ -998,88 +1331,158 @@ fip_signal_suspend(void *cls, * Create SUSPEND event for the given publish operation * and then clean up our state (without stop signal). * - * @param cls the 'struct GNUNET_FS_PublishContext' to signal for + * @param cls the `struct GNUNET_FS_PublishContext` to signal for */ void GNUNET_FS_publish_signal_suspend_ (void *cls) { struct GNUNET_FS_PublishContext *pc = cls; - if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task) - { - GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task); - pc->upload_task = GNUNET_SCHEDULER_NO_TASK; - } - GNUNET_FS_file_information_inspect (pc->fi, - &fip_signal_suspend, - pc); + if (NULL != pc->upload_task) + { + GNUNET_SCHEDULER_cancel (pc->upload_task); + pc->upload_task = NULL; + } + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_suspend, pc); + suspend_operation (pc->fi, pc); GNUNET_FS_end_top (pc->h, pc->top); + pc->top = NULL; publish_cleanup (pc); } + +/** + * We have gotten a reply for our space reservation request. + * Either fail (insufficient space) or start publishing for good. + * + * @param cls the `struct GNUNET_FS_PublishContext *` + * @param success positive reservation ID on success + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message on error, otherwise NULL + */ +static void +finish_reserve (void *cls, + int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + pc->qre = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Reservation complete (%d)!\n", + success); + if ((msg != NULL) || (success <= 0)) + { + GNUNET_asprintf (&pc->fi->emsg, + _("Datastore failure: %s"), + msg); + signal_publish_error (pc->fi, pc, pc->fi->emsg); + return; + } + pc->rid = success; + GNUNET_assert (NULL == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); +} + + +/** + * Calculate the total size of all of the files in the directory structure. + * + * @param fi file structure to traverse + */ +static uint64_t +compute_contents_size (struct GNUNET_FS_FileInformation *fi) +{ + struct GNUNET_FS_FileInformation *ent; + + if (GNUNET_YES != fi->is_directory) + return fi->data.file.file_size; + fi->data.dir.contents_size = 0; + for (ent = fi->data.dir.entries; NULL != ent; ent = ent->next) + fi->data.dir.contents_size += compute_contents_size (ent); + return fi->data.dir.contents_size; +} + + /** * Publish a file or directory. * * @param h handle to the file sharing subsystem * @param fi information about the file or directory structure to publish - * @param namespace namespace to publish the file in, NULL for no namespace + * @param ns namespace to publish the file in, NULL for no namespace * @param nid identifier to use for the publishd content in the namespace * (can be NULL, must be NULL if namespace is NULL) - * @param nuid update-identifier that will be used for future updates + * @param nuid update-identifier that will be used for future updates * (can be NULL, must be NULL if namespace or nid is NULL) - * @param options options for the publication + * @param options options for the publication * @return context that can be used to control the publish operation */ struct GNUNET_FS_PublishContext * GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_FileInformation *fi, - struct GNUNET_FS_Namespace *namespace, + struct GNUNET_FS_FileInformation *fi, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, const char *nid, - const char *nuid, - enum GNUNET_FS_PublishOptions options) + const char *nuid, + enum GNUNET_FS_PublishOptions options) { struct GNUNET_FS_PublishContext *ret; struct GNUNET_DATASTORE_Handle *dsh; + GNUNET_assert (NULL != h); + compute_contents_size (fi); if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - dsh = GNUNET_DATASTORE_connect (h->cfg, - h->sched); - if (NULL == dsh) - return NULL; - } + { + dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == dsh) + return NULL; + } else - { - dsh = NULL; - } - ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); + { + dsh = NULL; + } + ret = GNUNET_new (struct GNUNET_FS_PublishContext); ret->dsh = dsh; ret->h = h; ret->fi = fi; - ret->namespace = namespace; - if (namespace != NULL) - { - namespace->rc++; - GNUNET_assert (NULL != nid); - ret->nid = GNUNET_strdup (nid); - if (NULL != nuid) - ret->nuid = GNUNET_strdup (nuid); - } + if (NULL != ns) + { + ret->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey); + *ret->ns = *ns; + GNUNET_assert (NULL != nid); + ret->nid = GNUNET_strdup (nid); + if (NULL != nuid) + ret->nuid = GNUNET_strdup (nuid); + } + ret->options = options; /* signal start */ - GNUNET_FS_file_information_inspect (ret->fi, - &fip_signal_start, - ret); + GNUNET_FS_file_information_inspect (ret->fi, &fip_signal_start, ret); ret->fi_pos = ret->fi; ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret); GNUNET_FS_publish_sync_ (ret); - // FIXME: calculate space needed for "fi" - // and reserve as first task (then trigger - // "publish_main" from that continuation)! - ret->upload_task - = GNUNET_SCHEDULER_add_with_priority (h->sched, - GNUNET_SCHEDULER_PRIORITY_BACKGROUND, - &GNUNET_FS_publish_main_, - ret); + if (NULL != ret->dsh) + { + GNUNET_assert (NULL == ret->qre); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Reserving space for %u entries and %llu bytes for publication\n"), + (unsigned int) ret->reserve_entries, + (unsigned long long) ret->reserve_space); + ret->qre = + GNUNET_DATASTORE_reserve (ret->dsh, ret->reserve_space, + ret->reserve_entries, + &finish_reserve, + ret); + } + else + { + GNUNET_assert (NULL == ret->upload_task); + ret->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, ret); + } return ret; } @@ -1088,591 +1491,115 @@ GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, * Signal the FS's progress function that we are stopping * an upload. * - * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param cls closure (of type `struct GNUNET_FS_PublishContext *`) * @param fi the entry in the publish-structure * @param length length of the file or directory * @param meta metadata for the file or directory (can be modified) * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param anonymity pointer to selected anonymity level (can be modified) - * @param priority pointer to selected priority (can be modified) - * @param expirationTime pointer to selected expiration time (can be modified) + * @param bo block options (can be modified) + * @param do_index should we index? * @param client_info pointer to client context set upon creation (can be modified) - * @return GNUNET_OK to continue (always) + * @return #GNUNET_OK to continue (always) */ static int -fip_signal_stop(void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_CONTAINER_MetaData *meta, - struct GNUNET_FS_Uri **uri, - uint32_t *anonymity, - uint32_t *priority, - struct GNUNET_TIME_Absolute *expirationTime, - void **client_info) +fip_signal_stop (void *cls, + struct GNUNET_FS_FileInformation *fi, + uint64_t length, + struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, + int *do_index, void **client_info) { - struct GNUNET_FS_PublishContext*sc = cls; + struct GNUNET_FS_PublishContext *pc = cls; struct GNUNET_FS_ProgressInfo pi; uint64_t off; - if (fi->serialization != NULL) - { - GNUNET_FS_remove_sync_file_ (sc->h, - GNUNET_FS_SYNC_PATH_FILE_INFO, - fi->serialization); - GNUNET_free (fi->serialization); - fi->serialization = NULL; - } + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) + { + /* process entries in directory first */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_stop, pc); + } + if (NULL != fi->serialization) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + GNUNET_free (fi->serialization); + fi->serialization = NULL; + } off = (fi->chk_uri == NULL) ? 0 : length; pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; - GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off)); + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); *client_info = NULL; return GNUNET_OK; } /** - * Stop an upload. Will abort incomplete uploads (but + * Stop an upload. Will abort incomplete uploads (but * not remove blocks that have already been publishd) or * simply clean up the state for completed uploads. * Must NOT be called from within the event callback! * * @param pc context for the upload to stop */ -void +void GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc) { + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Publish stop called\n"); GNUNET_FS_end_top (pc->h, pc->top); - if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task) - { - GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task); - pc->upload_task = GNUNET_SCHEDULER_NO_TASK; - } - if (pc->serialization != NULL) - { - GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization); - GNUNET_free (pc->serialization); - pc->serialization = NULL; - } - GNUNET_FS_file_information_inspect (pc->fi, - &fip_signal_stop, - pc); - if (GNUNET_YES == pc->in_network_wait) - { - pc->in_network_wait = GNUNET_SYSERR; - return; - } + if (NULL != pc->ksk_pc) + { + GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); + pc->ksk_pc = NULL; + } + if (NULL != pc->sks_pc) + { + GNUNET_FS_publish_sks_cancel (pc->sks_pc); + pc->sks_pc = NULL; + } + if (NULL != pc->upload_task) + { + GNUNET_SCHEDULER_cancel (pc->upload_task); + pc->upload_task = NULL; + } + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_stop, pc); + + if (NULL != pc->fi->serialization) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + pc->fi->serialization); + GNUNET_free (pc->fi->serialization); + pc->fi->serialization = NULL; + } + off = (NULL == pc->fi->chk_uri) ? 0 : GNUNET_ntohll (pc->fi->chk_uri->data.chk.file_length); + + if (NULL != pc->serialization) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + GNUNET_free (pc->serialization); + pc->serialization = NULL; + } + if (NULL != pc->qre) + { + GNUNET_DATASTORE_cancel (pc->qre); + pc->qre = NULL; + } + pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi, off)); publish_cleanup (pc); } -/** - * Context for the KSK publication. - */ -struct PublishKskContext -{ - - /** - * Keywords to use. - */ - struct GNUNET_FS_Uri *ksk_uri; - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * The master block that we are sending - * (in plaintext), has "mdsize+slen" more - * bytes than the struct would suggest. - */ - struct KBlock *kb; - - /** - * Buffer of the same size as "kb" for - * the encrypted version. - */ - struct KBlock *cpy; - - /** - * Handle to the datastore, NULL if we are just - * simulating. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Function to call once we're done. - */ - GNUNET_FS_PublishContinuation cont; - - /** - * Closure for cont. - */ - void *cont_cls; - - /** - * When should the KBlocks expire? - */ - struct GNUNET_TIME_Absolute expirationTime; - - /** - * Size of the serialized metadata. - */ - ssize_t mdsize; - - /** - * Size of the (CHK) URI as a string. - */ - size_t slen; - - /** - * Keyword that we are currently processing. - */ - unsigned int i; - - /** - * Anonymity level for the KBlocks. - */ - uint32_t anonymity; - - /** - * Priority for the KBlocks. - */ - uint32_t priority; -}; - - -/** - * Continuation of "GNUNET_FS_publish_ksk" that performs - * the actual publishing operation (iterating over all - * of the keywords). - * - * @param cls closure of type "struct PublishKskContext*" - * @param tc unused - */ -static void -publish_ksk_cont (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - -/** - * Function called by the datastore API with - * the result from the PUT request. - * - * @param cls closure of type "struct PublishKskContext*" - * @param success GNUNET_OK on success - * @param msg error message (or NULL) - */ -static void -kb_put_cont (void *cls, - int success, - const char *msg) -{ - struct PublishKskContext *pkc = cls; - - if (GNUNET_OK != success) - { - GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); - GNUNET_free (pkc->cpy); - GNUNET_free (pkc->kb); - pkc->cont (pkc->cont_cls, - NULL, - msg); - GNUNET_FS_uri_destroy (pkc->ksk_uri); - GNUNET_free (pkc); - return; - } - GNUNET_SCHEDULER_add_continuation (pkc->h->sched, - &publish_ksk_cont, - pkc, - GNUNET_SCHEDULER_REASON_PREREQ_DONE); -} - - -/** - * Continuation of "GNUNET_FS_publish_ksk" that performs the actual - * publishing operation (iterating over all of the keywords). - * - * @param cls closure of type "struct PublishKskContext*" - * @param tc unused - */ -static void -publish_ksk_cont (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - struct PublishKskContext *pkc = cls; - const char *keyword; - GNUNET_HashCode key; - GNUNET_HashCode query; - struct GNUNET_CRYPTO_AesSessionKey skey; - struct GNUNET_CRYPTO_AesInitializationVector iv; - struct GNUNET_CRYPTO_RsaPrivateKey *pk; - - - if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) || - (NULL == pkc->dsh) ) - { - if (NULL != pkc->dsh) - GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); - GNUNET_free (pkc->cpy); - GNUNET_free (pkc->kb); - pkc->cont (pkc->cont_cls, - pkc->ksk_uri, - NULL); - GNUNET_FS_uri_destroy (pkc->ksk_uri); - GNUNET_free (pkc); - return; - } - keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++]; - /* first character of keyword indicates if it is - mandatory or not -- ignore for hashing */ - GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key); - GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv); - GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1], - pkc->slen + pkc->mdsize, - &skey, - &iv, - &pkc->cpy[1]); - pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key); - GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace); - GNUNET_CRYPTO_hash (&pkc->cpy->keyspace, - sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), - &query); - GNUNET_assert (GNUNET_OK == - GNUNET_CRYPTO_rsa_sign (pk, - &pkc->cpy->purpose, - &pkc->cpy->signature)); - GNUNET_CRYPTO_rsa_key_free (pk); - GNUNET_DATASTORE_put (pkc->dsh, - 0, - &query, - pkc->mdsize + - sizeof (struct KBlock) + - pkc->slen, - pkc->cpy, - GNUNET_BLOCK_TYPE_KBLOCK, - pkc->priority, - pkc->anonymity, - pkc->expirationTime, - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &kb_put_cont, - pkc); -} - - -/** - * Publish a CHK under various keywords on GNUnet. - * - * @param h handle to the file sharing subsystem - * @param ksk_uri keywords to use - * @param meta metadata to use - * @param uri URI to refer to in the KBlock - * @param expirationTime when the KBlock expires - * @param anonymity anonymity level for the KBlock - * @param priority priority for the KBlock - * @param options publication options - * @param cont continuation - * @param cont_cls closure for cont - */ -void -GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *ksk_uri, - const struct GNUNET_CONTAINER_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - struct GNUNET_TIME_Absolute expirationTime, - uint32_t anonymity, - uint32_t priority, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, - void *cont_cls) -{ - struct PublishKskContext *pkc; - char *uris; - size_t size; - char *kbe; - char *sptr; - - pkc = GNUNET_malloc (sizeof (struct PublishKskContext)); - pkc->h = h; - pkc->expirationTime = expirationTime; - pkc->anonymity = anonymity; - pkc->priority = priority; - pkc->cont = cont; - pkc->cont_cls = cont_cls; - if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - pkc->dsh = GNUNET_DATASTORE_connect (h->cfg, - h->sched); - if (pkc->dsh == NULL) - { - cont (cont_cls, NULL, _("Could not connect to datastore.")); - GNUNET_free (pkc); - return; - } - } - if (meta == NULL) - pkc->mdsize = 0; - else - pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); - GNUNET_assert (pkc->mdsize >= 0); - uris = GNUNET_FS_uri_to_string (uri); - pkc->slen = strlen (uris) + 1; - size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen; - if (size > MAX_KBLOCK_SIZE) - { - size = MAX_KBLOCK_SIZE; - pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen; - } - pkc->kb = GNUNET_malloc (size); - kbe = (char *) &pkc->kb[1]; - memcpy (kbe, uris, pkc->slen); - GNUNET_free (uris); - sptr = &kbe[pkc->slen]; - if (meta != NULL) - pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, - &sptr, - pkc->mdsize, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); - if (pkc->mdsize == -1) - { - GNUNET_break (0); - GNUNET_free (pkc->kb); - if (pkc->dsh != NULL) - GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); - cont (cont_cls, NULL, _("Internal error.")); - GNUNET_free (pkc); - return; - } - size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize; - - pkc->cpy = GNUNET_malloc (size); - pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) + - sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) + - pkc->mdsize + - pkc->slen); - pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK); - pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri); - GNUNET_SCHEDULER_add_continuation (h->sched, - &publish_ksk_cont, - pkc, - GNUNET_SCHEDULER_REASON_PREREQ_DONE); -} - - -/** - * Context for the SKS publication. - */ -struct PublishSksContext -{ - - /** - * Global FS context. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Handle to the datastore. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Function to call once we're done. - */ - GNUNET_FS_PublishContinuation cont; - - /** - * Closure for cont. - */ - void *cont_cls; - -}; - - -/** - * Function called by the datastore API with - * the result from the PUT (SBlock) request. - * - * @param cls closure of type "struct PublishSksContext*" - * @param success GNUNET_OK on success - * @param msg error message (or NULL) - */ -static void -sb_put_cont (void *cls, - int success, - const char *msg) -{ - struct PublishSksContext *psc = cls; - - if (NULL != psc->dsh) - GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO); - if (GNUNET_OK != success) - psc->cont (psc->cont_cls, - NULL, - msg); - else - psc->cont (psc->cont_cls, - psc->uri, - NULL); - GNUNET_FS_uri_destroy (psc->uri); - GNUNET_free (psc); -} - - -/** - * Publish an SBlock on GNUnet. - * - * @param h handle to the file sharing subsystem - * @param namespace namespace to publish in - * @param identifier identifier to use - * @param update update identifier to use - * @param meta metadata to use - * @param uri URI to refer to in the SBlock - * @param expirationTime when the SBlock expires - * @param anonymity anonymity level for the SBlock - * @param priority priority for the SBlock - * @param options publication options - * @param cont continuation - * @param cont_cls closure for cont - */ -void -GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_Namespace *namespace, - const char *identifier, - const char *update, - const struct GNUNET_CONTAINER_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - struct GNUNET_TIME_Absolute expirationTime, - uint32_t anonymity, - uint32_t priority, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, - void *cont_cls) -{ - struct PublishSksContext *psc; - struct GNUNET_CRYPTO_AesSessionKey sk; - struct GNUNET_CRYPTO_AesInitializationVector iv; - struct GNUNET_FS_Uri *sks_uri; - char *uris; - size_t size; - size_t slen; - size_t nidlen; - size_t idlen; - ssize_t mdsize; - struct SBlock *sb; - struct SBlock *sb_enc; - char *dest; - struct GNUNET_CONTAINER_MetaData *mmeta; - GNUNET_HashCode key; /* hash of thisId = key */ - GNUNET_HashCode id; /* hash of hc = identifier */ - GNUNET_HashCode query; /* id ^ nsid = DB query */ - - if (NULL == meta) - mmeta = GNUNET_CONTAINER_meta_data_create (); - else - mmeta = GNUNET_CONTAINER_meta_data_duplicate (meta); - uris = GNUNET_FS_uri_to_string (uri); - slen = strlen (uris) + 1; - idlen = strlen (identifier); - if (update == NULL) - update = ""; - nidlen = strlen (update) + 1; - mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (mmeta); - size = sizeof (struct SBlock) + slen + nidlen + mdsize; - if (size > MAX_SBLOCK_SIZE) - { - size = MAX_SBLOCK_SIZE; - mdsize = size - (sizeof (struct SBlock) + slen + nidlen); - } - sb = GNUNET_malloc (sizeof (struct SBlock) + size); - dest = (char *) &sb[1]; - memcpy (dest, update, nidlen); - dest += nidlen; - memcpy (dest, uris, slen); - GNUNET_free (uris); - dest += slen; - mdsize = GNUNET_CONTAINER_meta_data_serialize (mmeta, - &dest, - mdsize, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); - GNUNET_CONTAINER_meta_data_destroy (mmeta); - if (mdsize == -1) - { - GNUNET_break (0); - GNUNET_free (sb); - cont (cont_cls, - NULL, - _("Internal error.")); - return; - } - size = sizeof (struct SBlock) + mdsize + slen + nidlen; - sb_enc = GNUNET_malloc (size); - GNUNET_CRYPTO_hash (identifier, idlen, &key); - GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &id); - sks_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); - sks_uri->type = sks; - GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &sb_enc->subspace); - GNUNET_CRYPTO_hash (&sb_enc->subspace, - sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), - &sks_uri->data.sks.namespace); - sks_uri->data.sks.identifier = GNUNET_strdup (identifier); - GNUNET_CRYPTO_hash_xor (&id, - &sks_uri->data.sks.namespace, - &sb_enc->identifier); - GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv); - GNUNET_CRYPTO_aes_encrypt (&sb[1], - size - sizeof (struct SBlock), - &sk, - &iv, - &sb_enc[1]); - sb_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK); - sb_enc->purpose.size = htonl(slen + mdsize + nidlen - + sizeof(struct SBlock) - - sizeof(struct GNUNET_CRYPTO_RsaSignature)); - GNUNET_assert (GNUNET_OK == - GNUNET_CRYPTO_rsa_sign (namespace->key, - &sb_enc->purpose, - &sb_enc->signature)); - psc = GNUNET_malloc (sizeof(struct PublishSksContext)); - psc->uri = sks_uri; - psc->cont = cont; - psc->cont_cls = cont_cls; - if (0 != (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) - { - GNUNET_free (sb_enc); - GNUNET_free (sb); - sb_put_cont (psc, - GNUNET_OK, - NULL); - return; - } - psc->dsh = GNUNET_DATASTORE_connect (h->cfg, h->sched); - if (NULL == psc->dsh) - { - GNUNET_free (sb_enc); - GNUNET_free (sb); - sb_put_cont (psc, - GNUNET_NO, - _("Failed to connect to datastore.")); - return; - } - GNUNET_CRYPTO_hash_xor (&sks_uri->data.sks.namespace, - &id, - &query); - GNUNET_DATASTORE_put (psc->dsh, - 0, - &sb_enc->identifier, - size, - sb_enc, - GNUNET_BLOCK_TYPE_SBLOCK, - priority, - anonymity, - expirationTime, - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &sb_put_cont, - psc); - - GNUNET_free (sb); - GNUNET_free (sb_enc); -} - /* end of fs_publish.c */