2 This file is part of GNUnet.
3 Copyright (C) 2009, 2010 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
16 * @file fs/fs_publish.c
17 * @brief publish a file or directory in GNUnet
18 * @see https://gnunet.org/encoding
19 * @author Krista Bennett
20 * @author Christian Grothoff
23 #include "gnunet_constants.h"
24 #include "gnunet_signatures.h"
25 #include "gnunet_util_lib.h"
26 #include "gnunet_fs_service.h"
32 * Fill in all of the generic fields for
33 * a publish event and call the callback.
35 * @param pi structure to fill in
36 * @param pc overall publishing context
37 * @param p file information for the file being published
38 * @param offset where in the file are we so far
39 * @return value returned from callback
42 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
43 struct GNUNET_FS_PublishContext *pc,
44 const struct GNUNET_FS_FileInformation *p,
47 pi->value.publish.pc = pc;
48 pi->value.publish.fi = p;
49 pi->value.publish.cctx = p->client_info;
50 pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info;
51 pi->value.publish.filename = p->filename;
52 pi->value.publish.size =
53 (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
54 pi->value.publish.eta =
55 GNUNET_TIME_calculate_eta (p->start_time, offset,
56 pi->value.publish.size);
57 pi->value.publish.completed = offset;
58 pi->value.publish.duration =
59 GNUNET_TIME_absolute_get_duration (p->start_time);
60 pi->value.publish.anonymity = p->bo.anonymity_level;
62 return pc->h->upcb (pc->h->upcb_cls, pi);
67 * Cleanup the publish context, we're done with it.
69 * @param pc struct to clean up
72 publish_cleanup (struct GNUNET_FS_PublishContext *pc)
74 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
75 "Cleaning up publish context (done!)\n");
78 GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
81 GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
82 GNUNET_free_non_null (pc->nid);
83 GNUNET_free_non_null (pc->nuid);
84 GNUNET_free_non_null (pc->serialization);
87 GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
92 GNUNET_MQ_destroy (pc->mq);
95 GNUNET_assert (NULL == pc->upload_task);
101 * Function called by the datastore API with
102 * the result from the PUT request.
104 * @param cls the `struct GNUNET_FS_PublishContext *`
105 * @param success #GNUNET_OK on success
106 * @param min_expiration minimum expiration time required for content to be stored
107 * @param msg error message (or NULL)
110 ds_put_cont (void *cls,
112 struct GNUNET_TIME_Absolute min_expiration,
115 struct GNUNET_FS_PublishContext *pc = cls;
116 struct GNUNET_FS_ProgressInfo pi;
119 if (GNUNET_SYSERR == success)
121 GNUNET_asprintf (&pc->fi_pos->emsg,
122 _("Publishing failed: %s"),
124 pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
125 pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
126 pi.value.publish.specifics.error.message = pc->fi_pos->emsg;
127 pc->fi_pos->client_info =
128 GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0);
129 if ((GNUNET_YES != pc->fi_pos->is_directory) &&
130 (NULL != pc->fi_pos->filename) &&
131 (GNUNET_YES == pc->any_done) &&
132 (GNUNET_YES == pc->fi_pos->data.file.do_index))
134 /* run unindex to clean up */
135 GNUNET_FS_unindex_start (pc->h,
136 pc->fi_pos->filename,
141 pc->any_done = GNUNET_YES;
142 GNUNET_assert (NULL == pc->upload_task);
144 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
145 &GNUNET_FS_publish_main_, pc);
150 * Generate the callback that signals clients
151 * that a file (or directory) has been completely
154 * @param p the completed upload
155 * @param pc context of the publication
158 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
159 struct GNUNET_FS_PublishContext *pc)
161 struct GNUNET_FS_ProgressInfo pi;
163 pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
164 pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
165 pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
166 pi.value.publish.specifics.completed.sks_uri = p->sks_uri;
168 GNUNET_FS_publish_make_status_ (&pi, pc, p,
169 p->data.file.file_size);
174 * Generate the callback that signals clients
175 * that a file (or directory) has encountered
176 * a problem during publication.
178 * @param p the upload that had trouble
179 * @param pc context of the publication
180 * @param emsg error message
183 signal_publish_error (struct GNUNET_FS_FileInformation *p,
184 struct GNUNET_FS_PublishContext *pc,
187 struct GNUNET_FS_ProgressInfo pi;
189 p->emsg = GNUNET_strdup (emsg);
190 pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
191 pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
192 pi.value.publish.specifics.error.message = emsg;
193 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
194 if ((p->is_directory != GNUNET_YES) &&
195 (NULL != p->filename) &&
196 (GNUNET_YES == pc->any_done) &&
197 (p->data.file.do_index == GNUNET_YES))
199 /* run unindex to clean up */
200 GNUNET_FS_unindex_start (pc->h,
209 * Datastore returns from reservation cancel request.
211 * @param cls the `struct GNUNET_FS_PublishContext *`
212 * @param success success code (not used)
213 * @param min_expiration minimum expiration time required for content to be stored
214 * @param msg error message (typically NULL, not used)
217 finish_release_reserve (void *cls, int success,
218 struct GNUNET_TIME_Absolute min_expiration,
221 struct GNUNET_FS_PublishContext *pc = cls;
224 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
225 "Releasing reserve done!\n");
226 signal_publish_completion (pc->fi, pc);
227 pc->all_done = GNUNET_YES;
228 GNUNET_FS_publish_sync_ (pc);
233 * We've finished publishing the SBlock as part of a larger upload.
234 * Check the result and complete the larger upload.
236 * @param cls the `struct GNUNET_FS_PublishContext *` of the larger upload
237 * @param uri URI of the published SBlock
238 * @param emsg NULL on success, otherwise error message
241 publish_sblocks_cont (void *cls,
242 const struct GNUNET_FS_Uri *uri,
245 struct GNUNET_FS_PublishContext *pc = cls;
250 signal_publish_error (pc->fi, pc, emsg);
251 GNUNET_FS_publish_sync_ (pc);
256 /* sks publication, remember namespace URI */
257 pc->fi->sks_uri = GNUNET_FS_uri_dup (uri);
259 GNUNET_assert (pc->qre == NULL);
260 if ((pc->dsh != NULL) && (pc->rid != 0))
263 GNUNET_DATASTORE_release_reserve (pc->dsh, pc->rid, UINT_MAX, UINT_MAX,
264 &finish_release_reserve, pc);
268 finish_release_reserve (pc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
274 * We are almost done publishing the structure,
275 * add SBlocks (if needed).
277 * @param pc overall upload data
280 publish_sblock (struct GNUNET_FS_PublishContext *pc)
283 pc->sks_pc = GNUNET_FS_publish_sks (pc->h,
291 &publish_sblocks_cont, pc);
293 publish_sblocks_cont (pc, NULL, NULL);
298 * We've finished publishing a KBlock as part of a larger upload.
299 * Check the result and continue the larger upload.
301 * @param cls the `struct GNUNET_FS_PublishContext *`
302 * of the larger upload
303 * @param uri URI of the published blocks
304 * @param emsg NULL on success, otherwise error message
307 publish_kblocks_cont (void *cls,
308 const struct GNUNET_FS_Uri *uri,
311 struct GNUNET_FS_PublishContext *pc = cls;
312 struct GNUNET_FS_FileInformation *p = pc->fi_pos;
317 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
318 "Error uploading KSK blocks: %s\n",
320 signal_publish_error (p, pc, emsg);
321 GNUNET_FS_file_information_sync_ (p);
322 GNUNET_FS_publish_sync_ (pc);
323 GNUNET_assert (NULL == pc->upload_task);
325 GNUNET_SCHEDULER_add_with_priority
326 (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
327 &GNUNET_FS_publish_main_,
331 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
332 "KSK blocks published, moving on to next file\n");
334 signal_publish_completion (p, pc);
335 /* move on to next file */
337 pc->fi_pos = p->next;
340 GNUNET_FS_publish_sync_ (pc);
341 GNUNET_assert (NULL == pc->upload_task);
343 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
344 &GNUNET_FS_publish_main_, pc);
349 * Function called by the tree encoder to obtain
350 * a block of plaintext data (for the lowest level
353 * @param cls our publishing context
354 * @param offset identifies which block to get
355 * @param max (maximum) number of bytes to get; returning
356 * fewer will also cause errors
357 * @param buf where to copy the plaintext buffer
358 * @param emsg location to store an error message (on error)
359 * @return number of bytes copied to buf, 0 on error
362 block_reader (void *cls,
368 struct GNUNET_FS_PublishContext *pc = cls;
369 struct GNUNET_FS_FileInformation *p;
374 if (GNUNET_YES == p->is_directory)
376 pt_size = GNUNET_MIN (max, p->data.dir.dir_size - offset);
377 dd = p->data.dir.dir_data;
378 GNUNET_memcpy (buf, &dd[offset], pt_size);
382 if (UINT64_MAX == offset)
384 if (&GNUNET_FS_data_reader_file_ == p->data.file.reader)
386 /* force closing the file to avoid keeping too many files open */
387 p->data.file.reader (p->data.file.reader_cls, offset, 0, NULL, NULL);
391 pt_size = GNUNET_MIN (max, p->data.file.file_size - offset);
393 return 0; /* calling reader with pt_size==0
394 * might free buf, so don't! */
396 p->data.file.reader (p->data.file.reader_cls, offset, pt_size, buf,
405 * The tree encoder has finished processing a
406 * file. Call it's finish method and deal with
409 * @param cls our publishing context
412 encode_cont (void *cls)
414 struct GNUNET_FS_PublishContext *pc = cls;
415 struct GNUNET_FS_FileInformation *p;
416 struct GNUNET_FS_ProgressInfo pi;
420 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
421 "Finished with tree encoder\n");
423 p->chk_uri = GNUNET_FS_tree_encoder_get_uri (p->te);
424 GNUNET_FS_file_information_sync_ (p);
425 GNUNET_FS_tree_encoder_finish (p->te, &emsg);
429 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
430 "Error during tree walk: %s\n",
432 GNUNET_asprintf (&p->emsg,
433 _("Publishing failed: %s"),
436 pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
437 pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
438 pi.value.publish.specifics.error.message = p->emsg;
439 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
443 /* final progress event */
444 GNUNET_assert (NULL != p->chk_uri);
445 flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
446 pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
447 pi.value.publish.specifics.progress.data = NULL;
448 pi.value.publish.specifics.progress.offset = flen;
449 pi.value.publish.specifics.progress.data_len = 0;
450 pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
451 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen);
453 /* continue with main */ /* continue with main */
454 GNUNET_assert (NULL == pc->upload_task);
456 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
457 &GNUNET_FS_publish_main_, pc);
462 * Function called asking for the current (encoded)
463 * block to be processed. After processing the
464 * client should either call #GNUNET_FS_tree_encoder_next
465 * or (on error) #GNUNET_FS_tree_encoder_finish.
468 * @param chk content hash key for the block
469 * @param offset offset of the block in the file
470 * @param depth depth of the block in the file, 0 for DBLOCK
471 * @param type type of the block (IBLOCK or DBLOCK)
472 * @param block the (encrypted) block
473 * @param block_size size of @a block (in bytes)
476 block_proc (void *cls,
477 const struct ContentHashKey *chk,
480 enum GNUNET_BLOCK_Type type,
484 struct GNUNET_FS_PublishContext *pc = cls;
485 struct GNUNET_FS_FileInformation *p;
486 struct OnDemandBlock odb;
491 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
492 "Waiting for datastore connection\n");
493 GNUNET_assert (NULL == pc->upload_task);
495 GNUNET_SCHEDULER_add_with_priority
496 (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc);
500 if ( (GNUNET_YES != p->is_directory) &&
501 (GNUNET_YES == p->data.file.do_index) &&
502 (GNUNET_BLOCK_TYPE_FS_DBLOCK == type) )
504 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
505 "Indexing block `%s' for offset %llu with index size %u\n",
506 GNUNET_h2s (&chk->query),
507 (unsigned long long) offset,
508 (unsigned int) sizeof (struct OnDemandBlock));
509 odb.offset = GNUNET_htonll (offset);
510 odb.file_id = p->data.file.file_id;
511 GNUNET_assert (pc->qre == NULL);
513 GNUNET_DATASTORE_put (pc->dsh,
514 (p->is_directory == GNUNET_YES) ? 0 : pc->rid,
516 sizeof (struct OnDemandBlock),
518 GNUNET_BLOCK_TYPE_FS_ONDEMAND,
519 p->bo.content_priority,
520 p->bo.anonymity_level,
521 p->bo.replication_level,
522 p->bo.expiration_time,
527 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
528 "Publishing block `%s' for offset %llu with size %u\n",
529 GNUNET_h2s (&chk->query),
530 (unsigned long long) offset,
531 (unsigned int) block_size);
532 GNUNET_assert (pc->qre == NULL);
534 GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : pc->rid,
539 p->bo.content_priority,
540 p->bo.anonymity_level,
541 p->bo.replication_level,
542 p->bo.expiration_time,
550 * Function called with information about our
551 * progress in computing the tree encoding.
554 * @param offset where are we in the file
555 * @param pt_block plaintext of the currently processed block
556 * @param pt_size size of @a pt_block
557 * @param depth depth of the block in the tree, 0 for DBLOCK
560 progress_proc (void *cls, uint64_t offset,
561 const void *pt_block,
565 struct GNUNET_FS_PublishContext *pc = cls;
566 struct GNUNET_FS_FileInformation *p;
567 struct GNUNET_FS_FileInformation *par;
568 struct GNUNET_FS_ProgressInfo pi;
571 pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
572 pi.value.publish.specifics.progress.data = pt_block;
573 pi.value.publish.specifics.progress.offset = offset;
574 pi.value.publish.specifics.progress.data_len = pt_size;
575 pi.value.publish.specifics.progress.depth = depth;
576 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset);
578 (GNUNET_YES == p->is_directory) )
580 while (NULL != (par = p->dir))
583 GNUNET_assert (GNUNET_YES == par->is_directory);
584 p->data.dir.contents_completed += pt_size;
585 pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY;
586 pi.value.publish.specifics.progress_directory.completed = p->data.dir.contents_completed;
587 pi.value.publish.specifics.progress_directory.total = p->data.dir.contents_size;
588 pi.value.publish.specifics.progress_directory.eta = GNUNET_TIME_calculate_eta (p->start_time,
589 p->data.dir.contents_completed,
590 p->data.dir.contents_size);
591 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
598 * We are uploading a file or directory; load (if necessary) the next
599 * block into memory, encrypt it and send it to the FS service. Then
600 * continue with the main task.
602 * @param pc overall upload data
605 publish_content (struct GNUNET_FS_PublishContext *pc)
607 struct GNUNET_FS_FileInformation *p;
609 struct GNUNET_FS_DirectoryBuilder *db;
610 struct GNUNET_FS_FileInformation *dirpos;
615 GNUNET_assert (NULL != p);
618 if (GNUNET_YES == p->is_directory)
620 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n");
621 db = GNUNET_FS_directory_builder_create (p->meta);
622 dirpos = p->data.dir.entries;
623 while (NULL != dirpos)
625 if (GNUNET_YES == dirpos->is_directory)
627 raw_data = dirpos->data.dir.dir_data;
628 dirpos->data.dir.dir_data = NULL;
633 if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
634 (dirpos->data.file.file_size > 0))
636 raw_data = GNUNET_malloc (dirpos->data.file.file_size);
638 if (dirpos->data.file.file_size !=
639 dirpos->data.file.reader (dirpos->data.file.reader_cls, 0,
640 dirpos->data.file.file_size, raw_data,
643 GNUNET_free_non_null (emsg);
644 GNUNET_free (raw_data);
647 dirpos->data.file.reader (dirpos->data.file.reader_cls, UINT64_MAX, 0, 0, NULL);
650 GNUNET_FS_directory_builder_add (db, dirpos->chk_uri, dirpos->meta,
652 GNUNET_free_non_null (raw_data);
653 dirpos = dirpos->next;
655 GNUNET_free_non_null (p->data.dir.dir_data);
656 p->data.dir.dir_data = NULL;
657 p->data.dir.dir_size = 0;
658 GNUNET_FS_directory_builder_finish (db, &p->data.dir.dir_size,
659 &p->data.dir.dir_data);
660 GNUNET_FS_file_information_sync_ (p);
662 size = (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
663 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
664 "Creating tree encoder\n");
666 GNUNET_FS_tree_encoder_create (pc->h, size, pc, &block_reader,
667 &block_proc, &progress_proc,
671 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
672 "Processing next block from tree\n");
673 GNUNET_FS_tree_encoder_next (p->te);
678 * Check the response from the "fs" service to our 'start index'
681 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
682 * @param msg the response we got
685 check_index_start_failed (void *cls,
686 const struct GNUNET_MessageHeader *msg)
688 size_t msize = ntohs (msg->size) - sizeof (*msg);
689 const char *emsg = (const char *) &msg[1];
691 if (emsg[msize - sizeof (struct GNUNET_MessageHeader) - 1] != '\0')
694 return GNUNET_SYSERR;
701 * Process the response from the "fs" service to our 'start index'
704 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
705 * @param msg the response we got
708 handle_index_start_failed (void *cls,
709 const struct GNUNET_MessageHeader *msg)
711 struct GNUNET_FS_PublishContext *pc = cls;
712 struct GNUNET_FS_FileInformation *p;
713 const char *emsg = (const char *) &msg[1];
715 GNUNET_MQ_destroy (pc->mq);
718 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
719 _("Can not index file `%s': %s. Will try to insert instead.\n"),
722 p->data.file.do_index = GNUNET_NO;
723 GNUNET_FS_file_information_sync_ (p);
724 publish_content (pc);
729 * Process the response from the "fs" service to our 'start index'
732 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
733 * @param msg the response we got
736 handle_index_start_ok (void *cls,
737 const struct GNUNET_MessageHeader *msg)
739 struct GNUNET_FS_PublishContext *pc = cls;
740 struct GNUNET_FS_FileInformation *p;
742 GNUNET_MQ_destroy (pc->mq);
745 p->data.file.index_start_confirmed = GNUNET_YES;
746 GNUNET_FS_file_information_sync_ (p);
747 publish_content (pc);
752 * Generic error handler, called with the appropriate error code and
753 * the same closure specified at the creation of the message queue.
754 * Not every message queue implementation supports an error handler.
756 * @param cls closure with the `struct GNUNET_FS_PublishContext *`
757 * @param error error code
760 index_mq_error_handler (void *cls,
761 enum GNUNET_MQ_Error error)
763 struct GNUNET_FS_PublishContext *pc = cls;
764 struct GNUNET_FS_FileInformation *p;
768 GNUNET_MQ_destroy (pc->mq);
772 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
773 _("Can not index file `%s': %s. Will try to insert instead.\n"),
775 _("error on index-start request to `fs' service"));
776 p->data.file.do_index = GNUNET_NO;
777 GNUNET_FS_file_information_sync_ (p);
778 publish_content (pc);
783 * Function called once the hash computation over an
784 * indexed file has completed.
786 * @param cls closure, our publishing context
787 * @param res resulting hash, NULL on error
790 hash_for_index_cb (void *cls,
791 const struct GNUNET_HashCode *res)
793 struct GNUNET_FS_PublishContext *pc = cls;
794 struct GNUNET_MQ_MessageHandler handlers[] = {
795 GNUNET_MQ_hd_fixed_size (index_start_ok,
796 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK,
797 struct GNUNET_MessageHeader,
799 GNUNET_MQ_hd_var_size (index_start_failed,
800 GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED,
801 struct GNUNET_MessageHeader,
803 GNUNET_MQ_handler_end ()
805 struct GNUNET_FS_FileInformation *p;
806 struct GNUNET_MQ_Envelope *env;
807 struct IndexStartMessage *ism;
817 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
818 _("Can not index file `%s': %s. Will try to insert instead.\n"),
820 _("failed to compute hash"));
821 p->data.file.do_index = GNUNET_NO;
822 GNUNET_FS_file_information_sync_ (p);
823 publish_content (pc);
826 if (GNUNET_YES == p->data.file.index_start_confirmed)
828 publish_content (pc);
831 fn = GNUNET_STRINGS_filename_expand (p->filename);
832 GNUNET_assert (fn != NULL);
833 slen = strlen (fn) + 1;
835 GNUNET_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage))
837 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
839 ("Can not index file `%s': %s. Will try to insert instead.\n"),
840 fn, _("filename too long"));
842 p->data.file.do_index = GNUNET_NO;
843 GNUNET_FS_file_information_sync_ (p);
844 publish_content (pc);
847 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
848 "Hash of indexed file `%s' is `%s'\n",
851 if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
853 p->data.file.file_id = *res;
854 p->data.file.have_hash = GNUNET_YES;
855 p->data.file.index_start_confirmed = GNUNET_YES;
856 GNUNET_FS_file_information_sync_ (p);
857 publish_content (pc);
861 pc->mq = GNUNET_CLIENT_connect (pc->h->cfg,
864 &index_mq_error_handler,
868 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
869 _("Can not index file `%s': %s. Will try to insert instead.\n"),
871 _("could not connect to `fs' service"));
872 p->data.file.do_index = GNUNET_NO;
873 publish_content (pc);
877 if (p->data.file.have_hash != GNUNET_YES)
879 p->data.file.file_id = *res;
880 p->data.file.have_hash = GNUNET_YES;
881 GNUNET_FS_file_information_sync_ (p);
883 env = GNUNET_MQ_msg_extra (ism,
885 GNUNET_MESSAGE_TYPE_FS_INDEX_START);
887 GNUNET_DISK_file_get_identifiers (p->filename,
891 ism->device = GNUNET_htonll (dev);
892 ism->inode = GNUNET_htonll (ino);
896 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
897 _("Failed to get file identifiers for `%s'\n"),
901 GNUNET_memcpy (&ism[1],
905 GNUNET_MQ_send (pc->mq,
911 * We've computed the CHK/LOC URI, now publish the KSKs (if applicable).
913 * @param pc publishing context to do this for
916 publish_kblocks (struct GNUNET_FS_PublishContext *pc)
918 struct GNUNET_FS_FileInformation *p;
921 /* upload of "p" complete, publish KBlocks! */
922 if (NULL != p->keywords)
924 pc->ksk_pc = GNUNET_FS_publish_ksk (pc->h,
930 &publish_kblocks_cont,
935 publish_kblocks_cont (pc, p->chk_uri, NULL);
941 * Process the response from the "fs" service to our LOC sign request.
943 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
944 * @param sig the response we got
947 handle_signature_response (void *cls,
948 const struct ResponseLocSignatureMessage *sig)
950 struct GNUNET_FS_PublishContext *pc = cls;
951 struct GNUNET_FS_FileInformation *p;
954 p->chk_uri->type = GNUNET_FS_URI_LOC;
955 /* p->data.loc.fi kept from CHK before */
956 p->chk_uri->data.loc.peer = sig->peer;
957 p->chk_uri->data.loc.expirationTime
958 = GNUNET_TIME_absolute_ntoh (sig->expiration_time);
959 p->chk_uri->data.loc.contentSignature = sig->signature;
960 GNUNET_FS_file_information_sync_ (p);
961 GNUNET_FS_publish_sync_ (pc);
962 publish_kblocks (pc);
967 * Generic error handler, called with the appropriate error code and
968 * the same closure specified at the creation of the message queue.
969 * Not every message queue implementation supports an error handler.
971 * @param cls closure with the `struct GNUNET_FS_PublishContext *`
972 * @param error error code
975 loc_mq_error_handler (void *cls,
976 enum GNUNET_MQ_Error error)
978 struct GNUNET_FS_PublishContext *pc = cls;
982 GNUNET_MQ_destroy (pc->mq);
985 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
986 _("Can not create LOC URI. Will continue with CHK instead.\n"));
987 publish_kblocks (pc);
992 * We're publishing without anonymity. Contact the FS service
993 * to create a signed LOC URI for further processing, then
994 * continue with KSKs.
996 * @param pc the publishing context do to this for
999 create_loc_uri (struct GNUNET_FS_PublishContext *pc)
1001 struct GNUNET_MQ_MessageHandler handlers[] = {
1002 GNUNET_MQ_hd_fixed_size (signature_response,
1003 GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE,
1004 struct ResponseLocSignatureMessage,
1006 GNUNET_MQ_handler_end ()
1008 struct GNUNET_MQ_Envelope *env;
1009 struct RequestLocSignatureMessage *req;
1010 struct GNUNET_FS_FileInformation *p;
1013 GNUNET_MQ_destroy (pc->mq);
1014 pc->mq = GNUNET_CLIENT_connect (pc->h->cfg,
1017 &loc_mq_error_handler,
1021 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1022 _("Can not create LOC URI. Will continue with CHK instead.\n"));
1023 publish_kblocks (pc);
1027 env = GNUNET_MQ_msg (req,
1028 GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN);
1029 req->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT);
1030 req->expiration_time = GNUNET_TIME_absolute_hton (p->bo.expiration_time);
1031 req->chk = p->chk_uri->data.chk.chk;
1032 req->file_length = GNUNET_htonll (p->chk_uri->data.chk.file_length);
1033 GNUNET_MQ_send (pc->mq,
1039 * Main function that performs the upload.
1041 * @param cls `struct GNUNET_FS_PublishContext *` identifies the upload
1044 GNUNET_FS_publish_main_ (void *cls)
1046 struct GNUNET_FS_PublishContext *pc = cls;
1047 struct GNUNET_FS_ProgressInfo pi;
1048 struct GNUNET_FS_FileInformation *p;
1051 pc->upload_task = NULL;
1055 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1056 "Publishing complete, now publishing SKS and KSK blocks.\n");
1057 /* upload of entire hierarchy complete,
1058 * publish namespace entries */
1059 GNUNET_FS_publish_sync_ (pc);
1060 publish_sblock (pc);
1063 /* find starting position */
1064 while ( (GNUNET_YES == p->is_directory) &&
1065 (NULL != p->data.dir.entries) &&
1066 (NULL == p->emsg) &&
1067 (NULL == p->data.dir.entries->chk_uri) )
1069 p = p->data.dir.entries;
1071 GNUNET_FS_publish_sync_ (pc);
1073 /* abort on error */
1074 if (NULL != p->emsg)
1076 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1077 "Error uploading: %s\n",
1079 /* error with current file, abort all
1080 * related files as well! */
1081 while (NULL != p->dir)
1083 fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
1084 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
1088 GNUNET_asprintf (&p->emsg,
1089 _("Recursive upload failed at `%s': %s"),
1096 GNUNET_asprintf (&p->emsg,
1097 _("Recursive upload failed: %s"),
1100 pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
1101 pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
1102 pi.value.publish.specifics.error.message = p->emsg;
1103 p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
1105 pc->all_done = GNUNET_YES;
1106 GNUNET_FS_publish_sync_ (pc);
1109 /* handle completion */
1110 if (NULL != p->chk_uri)
1112 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1113 "File upload complete, now publishing KSK blocks.\n");
1114 GNUNET_FS_publish_sync_ (pc);
1116 if ( (0 == p->bo.anonymity_level) &&
1118 GNUNET_FS_uri_test_loc (p->chk_uri)) )
1120 /* zero anonymity, box CHK URI in LOC URI */
1121 create_loc_uri (pc);
1125 publish_kblocks (pc);
1129 if ((GNUNET_YES != p->is_directory) && (p->data.file.do_index))
1131 if (NULL == p->filename)
1133 p->data.file.do_index = GNUNET_NO;
1134 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1135 _("Can not index file `%s': %s. Will try to insert instead.\n"),
1137 _("needs to be an actual file"));
1138 GNUNET_FS_file_information_sync_ (p);
1139 publish_content (pc);
1142 if (p->data.file.have_hash)
1144 hash_for_index_cb (pc, &p->data.file.file_id);
1148 p->start_time = GNUNET_TIME_absolute_get ();
1150 GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, p->filename,
1151 HASHING_BLOCKSIZE, &hash_for_index_cb, pc);
1155 publish_content (pc);
1160 * Signal the FS's progress function that we are starting
1163 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
1164 * @param fi the entry in the publish-structure
1165 * @param length length of the file or directory
1166 * @param meta metadata for the file or directory (can be modified)
1167 * @param uri pointer to the keywords that will be used for this entry (can be modified)
1168 * @param bo block options
1169 * @param do_index should we index?
1170 * @param client_info pointer to client context set upon creation (can be modified)
1171 * @return #GNUNET_OK to continue (always)
1174 fip_signal_start (void *cls,
1175 struct GNUNET_FS_FileInformation *fi,
1177 struct GNUNET_CONTAINER_MetaData *meta,
1178 struct GNUNET_FS_Uri **uri,
1179 struct GNUNET_FS_BlockOptions *bo,
1183 struct GNUNET_FS_PublishContext *pc = cls;
1184 struct GNUNET_FS_ProgressInfo pi;
1188 if (GNUNET_YES == pc->skip_next_fi_callback)
1190 pc->skip_next_fi_callback = GNUNET_NO;
1193 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1194 "Starting publish operation\n");
1197 /* space for on-demand blocks */
1198 pc->reserve_space +=
1199 ((length + DBLOCK_SIZE -
1200 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1204 /* space for DBlocks */
1205 pc->reserve_space += length;
1207 /* entries for IBlocks and DBlocks, space for IBlocks */
1211 left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1212 pc->reserve_entries += left;
1215 left = left * sizeof (struct ContentHashKey);
1216 pc->reserve_space += left;
1218 pc->reserve_entries++;
1219 /* entries and space for keywords */
1222 kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1223 pc->reserve_entries += kc;
1224 pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc;
1226 pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1227 *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
1228 GNUNET_FS_file_information_sync_ (fi);
1229 if ((fi->is_directory) && (fi->dir != NULL))
1231 /* We are a directory, and we are not top-level; process entries in directory */
1232 pc->skip_next_fi_callback = GNUNET_YES;
1233 GNUNET_FS_file_information_inspect (fi, &fip_signal_start, pc);
1240 * Actually signal the FS's progress function that we are suspending
1243 * @param fi the entry in the publish-structure
1244 * @param pc the publish context of which a file is being suspended
1247 suspend_operation (struct GNUNET_FS_FileInformation *fi,
1248 struct GNUNET_FS_PublishContext *pc)
1250 struct GNUNET_FS_ProgressInfo pi;
1253 if (NULL != pc->ksk_pc)
1255 GNUNET_FS_publish_ksk_cancel (pc->ksk_pc);
1258 if (NULL != pc->sks_pc)
1260 GNUNET_FS_publish_sks_cancel (pc->sks_pc);
1263 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1264 "Suspending publish operation\n");
1265 GNUNET_free_non_null (fi->serialization);
1266 fi->serialization = NULL;
1267 off = (NULL == fi->chk_uri) ? 0 : (GNUNET_YES == fi->is_directory) ? fi->data.dir.dir_size : fi->data.file.file_size;
1268 pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1269 GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1270 if (NULL != pc->qre)
1272 GNUNET_DATASTORE_cancel (pc->qre);
1275 if (NULL != pc->dsh)
1277 GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1285 * Signal the FS's progress function that we are suspending
1286 * an upload. Performs the recursion.
1288 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
1289 * @param fi the entry in the publish-structure
1290 * @param length length of the file or directory
1291 * @param meta metadata for the file or directory (can be modified)
1292 * @param uri pointer to the keywords that will be used for this entry (can be modified)
1293 * @param bo block options
1294 * @param do_index should we index?
1295 * @param client_info pointer to client context set upon creation (can be modified)
1296 * @return #GNUNET_OK to continue (always)
1299 fip_signal_suspend (void *cls,
1300 struct GNUNET_FS_FileInformation *fi,
1302 struct GNUNET_CONTAINER_MetaData *meta,
1303 struct GNUNET_FS_Uri **uri,
1304 struct GNUNET_FS_BlockOptions *bo,
1308 struct GNUNET_FS_PublishContext *pc = cls;
1310 if (GNUNET_YES == pc->skip_next_fi_callback)
1312 pc->skip_next_fi_callback = GNUNET_NO;
1315 if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta))
1317 /* process entries in directory */
1318 pc->skip_next_fi_callback = GNUNET_YES;
1319 GNUNET_FS_file_information_inspect (fi, &fip_signal_suspend, pc);
1321 suspend_operation (fi, pc);
1322 *client_info = NULL;
1328 * Create SUSPEND event for the given publish operation
1329 * and then clean up our state (without stop signal).
1331 * @param cls the `struct GNUNET_FS_PublishContext` to signal for
1334 GNUNET_FS_publish_signal_suspend_ (void *cls)
1336 struct GNUNET_FS_PublishContext *pc = cls;
1338 if (NULL != pc->upload_task)
1340 GNUNET_SCHEDULER_cancel (pc->upload_task);
1341 pc->upload_task = NULL;
1343 pc->skip_next_fi_callback = GNUNET_YES;
1344 GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_suspend, pc);
1345 suspend_operation (pc->fi, pc);
1346 GNUNET_FS_end_top (pc->h, pc->top);
1348 publish_cleanup (pc);
1353 * We have gotten a reply for our space reservation request.
1354 * Either fail (insufficient space) or start publishing for good.
1356 * @param cls the `struct GNUNET_FS_PublishContext *`
1357 * @param success positive reservation ID on success
1358 * @param min_expiration minimum expiration time required for content to be stored
1359 * @param msg error message on error, otherwise NULL
1362 finish_reserve (void *cls,
1364 struct GNUNET_TIME_Absolute min_expiration,
1367 struct GNUNET_FS_PublishContext *pc = cls;
1370 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1371 "Reservation complete (%d)!\n",
1373 if ((msg != NULL) || (success <= 0))
1375 GNUNET_asprintf (&pc->fi->emsg,
1376 _("Datastore failure: %s"),
1378 signal_publish_error (pc->fi, pc, pc->fi->emsg);
1382 GNUNET_assert (NULL == pc->upload_task);
1384 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1385 &GNUNET_FS_publish_main_, pc);
1390 * Calculate the total size of all of the files in the directory structure.
1392 * @param fi file structure to traverse
1395 compute_contents_size (struct GNUNET_FS_FileInformation *fi)
1397 struct GNUNET_FS_FileInformation *ent;
1399 if (GNUNET_YES != fi->is_directory)
1400 return fi->data.file.file_size;
1401 fi->data.dir.contents_size = 0;
1402 for (ent = fi->data.dir.entries; NULL != ent; ent = ent->next)
1403 fi->data.dir.contents_size += compute_contents_size (ent);
1404 return fi->data.dir.contents_size;
1409 * Publish a file or directory.
1411 * @param h handle to the file sharing subsystem
1412 * @param fi information about the file or directory structure to publish
1413 * @param ns namespace to publish the file in, NULL for no namespace
1414 * @param nid identifier to use for the publishd content in the namespace
1415 * (can be NULL, must be NULL if namespace is NULL)
1416 * @param nuid update-identifier that will be used for future updates
1417 * (can be NULL, must be NULL if namespace or nid is NULL)
1418 * @param options options for the publication
1419 * @return context that can be used to control the publish operation
1421 struct GNUNET_FS_PublishContext *
1422 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1423 struct GNUNET_FS_FileInformation *fi,
1424 const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
1427 enum GNUNET_FS_PublishOptions options)
1429 struct GNUNET_FS_PublishContext *ret;
1430 struct GNUNET_DATASTORE_Handle *dsh;
1432 GNUNET_assert (NULL != h);
1433 compute_contents_size (fi);
1434 if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1436 dsh = GNUNET_DATASTORE_connect (h->cfg);
1444 ret = GNUNET_new (struct GNUNET_FS_PublishContext);
1450 ret->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey);
1452 GNUNET_assert (NULL != nid);
1453 ret->nid = GNUNET_strdup (nid);
1455 ret->nuid = GNUNET_strdup (nuid);
1457 ret->options = options;
1459 GNUNET_FS_file_information_inspect (ret->fi, &fip_signal_start, ret);
1460 ret->fi_pos = ret->fi;
1461 ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1462 GNUNET_FS_publish_sync_ (ret);
1463 if (NULL != ret->dsh)
1465 GNUNET_assert (NULL == ret->qre);
1466 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1467 _("Reserving space for %u entries and %llu bytes for publication\n"),
1468 (unsigned int) ret->reserve_entries,
1469 (unsigned long long) ret->reserve_space);
1471 GNUNET_DATASTORE_reserve (ret->dsh, ret->reserve_space,
1472 ret->reserve_entries,
1478 GNUNET_assert (NULL == ret->upload_task);
1480 GNUNET_SCHEDULER_add_with_priority
1481 (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, ret);
1488 * Signal the FS's progress function that we are stopping
1491 * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
1492 * @param fi the entry in the publish-structure
1493 * @param length length of the file or directory
1494 * @param meta metadata for the file or directory (can be modified)
1495 * @param uri pointer to the keywords that will be used for this entry (can be modified)
1496 * @param bo block options (can be modified)
1497 * @param do_index should we index?
1498 * @param client_info pointer to client context set upon creation (can be modified)
1499 * @return #GNUNET_OK to continue (always)
1502 fip_signal_stop (void *cls,
1503 struct GNUNET_FS_FileInformation *fi,
1505 struct GNUNET_CONTAINER_MetaData *meta,
1506 struct GNUNET_FS_Uri **uri,
1507 struct GNUNET_FS_BlockOptions *bo,
1508 int *do_index, void **client_info)
1510 struct GNUNET_FS_PublishContext *pc = cls;
1511 struct GNUNET_FS_ProgressInfo pi;
1514 if (GNUNET_YES == pc->skip_next_fi_callback)
1516 pc->skip_next_fi_callback = GNUNET_NO;
1519 if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta))
1521 /* process entries in directory first */
1522 pc->skip_next_fi_callback = GNUNET_YES;
1523 GNUNET_FS_file_information_inspect (fi, &fip_signal_stop, pc);
1525 if (NULL != fi->serialization)
1527 GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO,
1529 GNUNET_free (fi->serialization);
1530 fi->serialization = NULL;
1532 off = (fi->chk_uri == NULL) ? 0 : length;
1533 pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1534 GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1535 *client_info = NULL;
1541 * Stop an upload. Will abort incomplete uploads (but
1542 * not remove blocks that have already been publishd) or
1543 * simply clean up the state for completed uploads.
1544 * Must NOT be called from within the event callback!
1546 * @param pc context for the upload to stop
1549 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1551 struct GNUNET_FS_ProgressInfo pi;
1554 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1555 "Publish stop called\n");
1556 GNUNET_FS_end_top (pc->h, pc->top);
1557 if (NULL != pc->ksk_pc)
1559 GNUNET_FS_publish_ksk_cancel (pc->ksk_pc);
1562 if (NULL != pc->sks_pc)
1564 GNUNET_FS_publish_sks_cancel (pc->sks_pc);
1567 if (NULL != pc->upload_task)
1569 GNUNET_SCHEDULER_cancel (pc->upload_task);
1570 pc->upload_task = NULL;
1572 pc->skip_next_fi_callback = GNUNET_YES;
1573 GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_stop, pc);
1575 if (NULL != pc->fi->serialization)
1577 GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO,
1578 pc->fi->serialization);
1579 GNUNET_free (pc->fi->serialization);
1580 pc->fi->serialization = NULL;
1582 off = (NULL == pc->fi->chk_uri) ? 0 : GNUNET_ntohll (pc->fi->chk_uri->data.chk.file_length);
1584 if (NULL != pc->serialization)
1586 GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH,
1588 GNUNET_free (pc->serialization);
1589 pc->serialization = NULL;
1591 if (NULL != pc->qre)
1593 GNUNET_DATASTORE_cancel (pc->qre);
1596 pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1597 GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi, off));
1598 publish_cleanup (pc);
1602 /* end of fs_publish.c */