2 This file is part of GNUnet.
3 Copyright (C) 2003--2013, 2016 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
17 * @file fs/fs_unindex.c
18 * @author Krista Grothoff
19 * @author Christian Grothoff
20 * @brief Unindex file.
23 #include "gnunet_constants.h"
24 #include "gnunet_fs_service.h"
25 #include "gnunet_protocols.h"
29 #include "fs_publish_ublock.h"
33 * Function called by the tree encoder to obtain
34 * a block of plaintext data (for the lowest level
37 * @param cls our publishing context
38 * @param offset identifies which block to get
39 * @param max (maximum) number of bytes to get; returning
40 * fewer will also cause errors
41 * @param buf where to copy the plaintext buffer
42 * @param emsg location to store an error message (on error)
43 * @return number of bytes copied to buf, 0 on error
46 unindex_reader (void *cls,
52 struct GNUNET_FS_UnindexContext *uc = cls;
55 pt_size = GNUNET_MIN (max, uc->file_size - offset);
56 if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET))
58 *emsg = GNUNET_strdup (_("Failed to find given position in file"));
61 if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size))
63 *emsg = GNUNET_strdup (_("Failed to read file"));
71 * Fill in all of the generic fields for
72 * an unindex event and call the callback.
74 * @param pi structure to fill in
75 * @param uc overall unindex context
76 * @param offset where we are in the file (for progress)
79 GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
80 struct GNUNET_FS_UnindexContext *uc,
83 pi->value.unindex.uc = uc;
84 pi->value.unindex.cctx = uc->client_info;
85 pi->value.unindex.filename = uc->filename;
86 pi->value.unindex.size = uc->file_size;
87 pi->value.unindex.eta =
88 GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size);
89 pi->value.unindex.duration =
90 GNUNET_TIME_absolute_get_duration (uc->start_time);
91 pi->value.unindex.completed = offset;
93 uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi);
98 * Function called with information about our
99 * progress in computing the tree encoding.
102 * @param offset where are we in the file
103 * @param pt_block plaintext of the currently processed block
104 * @param pt_size size of pt_block
105 * @param depth depth of the block in the tree, 0 for DBLOCK
108 unindex_progress (void *cls,
110 const void *pt_block,
114 struct GNUNET_FS_UnindexContext *uc = cls;
115 struct GNUNET_FS_ProgressInfo pi;
117 pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS;
118 pi.value.unindex.specifics.progress.data = pt_block;
119 pi.value.unindex.specifics.progress.offset = offset;
120 pi.value.unindex.specifics.progress.data_len = pt_size;
121 pi.value.unindex.specifics.progress.depth = depth;
122 GNUNET_FS_unindex_make_status_ (&pi, uc, offset);
127 * We've encountered an error during
128 * unindexing. Signal the client.
130 * @param uc context for the failed unindexing operation
133 signal_unindex_error (struct GNUNET_FS_UnindexContext *uc)
135 struct GNUNET_FS_ProgressInfo pi;
137 pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR;
138 pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
139 pi.value.unindex.specifics.error.message = uc->emsg;
140 GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
145 * Continuation called to notify client about result of the
146 * datastore removal operation.
149 * @param success #GNUNET_SYSERR on failure
150 * @param min_expiration minimum expiration time required for content to be stored
151 * @param msg NULL on success, otherwise an error message
154 process_cont (void *cls,
156 struct GNUNET_TIME_Absolute min_expiration,
159 struct GNUNET_FS_UnindexContext *uc = cls;
161 if (success == GNUNET_SYSERR)
163 uc->emsg = GNUNET_strdup (msg);
164 signal_unindex_error (uc);
167 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
168 "Datastore REMOVE operation succeeded\n");
169 GNUNET_FS_tree_encoder_next (uc->tc);
174 * Function called asking for the current (encoded)
175 * block to be processed. After processing the
176 * client should either call "GNUNET_FS_tree_encode_next"
177 * or (on error) "GNUNET_FS_tree_encode_finish".
180 * @param chk content hash key for the block (key for lookup in the datastore)
181 * @param offset offset of the block
182 * @param depth depth of the block, 0 for DBLOCK
183 * @param type type of the block (IBLOCK or DBLOCK)
184 * @param block the (encrypted) block
185 * @param block_size size of block (in bytes)
188 unindex_process (void *cls,
189 const struct ContentHashKey *chk,
192 enum GNUNET_BLOCK_Type type,
196 struct GNUNET_FS_UnindexContext *uc = cls;
199 struct OnDemandBlock odb;
201 if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK)
206 else /* on-demand encoded DBLOCK */
208 size = sizeof (struct OnDemandBlock);
209 odb.offset = GNUNET_htonll (offset);
210 odb.file_id = uc->file_id;
213 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
214 "Sending REMOVE request to DATASTORE service\n");
215 GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1,
222 * Function called with the response from the FS service to our
223 * unindexing request.
225 * @param cls closure, unindex context
226 * @param msg the response
229 handle_unindex_response (void *cls,
230 const struct GNUNET_MessageHeader *msg)
232 struct GNUNET_FS_UnindexContext *uc = cls;
233 struct GNUNET_FS_ProgressInfo pi;
237 GNUNET_MQ_destroy (uc->mq);
240 uc->state = UNINDEX_STATE_COMPLETE;
241 pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED;
242 pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
243 GNUNET_FS_unindex_sync_ (uc);
244 GNUNET_FS_unindex_make_status_ (&pi,
251 * Generic error handler, called with the appropriate error code and
252 * the same closure specified at the creation of the message queue.
253 * Not every message queue implementation supports an error handler.
255 * @param cls closure with the `struct GNUNET_FS_UnindexContext *`
256 * @param error error code
259 unindex_mq_error_handler (void *cls,
260 enum GNUNET_MQ_Error error)
262 struct GNUNET_FS_UnindexContext *uc = cls;
266 GNUNET_MQ_destroy (uc->mq);
269 uc->state = UNINDEX_STATE_ERROR;
270 uc->emsg = GNUNET_strdup (_("Error communicating with `fs' service."));
271 GNUNET_FS_unindex_sync_ (uc);
272 signal_unindex_error (uc);
277 * Function called when we are done with removing UBlocks.
278 * Disconnect from datastore and notify FS service about
281 * @param uc our unindexing context
284 unindex_finish (struct GNUNET_FS_UnindexContext *uc)
286 struct GNUNET_MQ_MessageHandler handlers[] = {
287 GNUNET_MQ_hd_fixed_size (unindex_response,
288 GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK,
289 struct GNUNET_MessageHeader,
291 GNUNET_MQ_handler_end ()
294 struct GNUNET_MQ_Envelope *env;
295 struct UnindexMessage *req;
297 /* generate final progress message */
298 unindex_progress (uc,
303 GNUNET_FS_tree_encoder_finish (uc->tc,
306 GNUNET_DISK_file_close (uc->fh);
308 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
310 uc->state = UNINDEX_STATE_FS_NOTIFY;
311 GNUNET_FS_unindex_sync_ (uc);
312 uc->mq = GNUNET_CLIENT_connect (uc->h->cfg,
315 &unindex_mq_error_handler,
319 uc->state = UNINDEX_STATE_ERROR;
321 GNUNET_strdup (_("Failed to connect to FS service for unindexing."));
322 GNUNET_FS_unindex_sync_ (uc);
323 signal_unindex_error (uc);
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Sending UNINDEX message to FS service\n");
328 env = GNUNET_MQ_msg (req,
329 GNUNET_MESSAGE_TYPE_FS_UNINDEX);
331 req->file_id = uc->file_id;
332 GNUNET_MQ_send (uc->mq,
338 * Function called by the directory scanner as we extract keywords
339 * that we will need to remove UBlocks.
341 * @param cls the 'struct GNUNET_FS_UnindexContext *'
342 * @param filename which file we are making progress on
343 * @param is_directory #GNUNET_YES if this is a directory,
344 * #GNUNET_NO if this is a file
345 * #GNUNET_SYSERR if it is neither (or unknown)
346 * @param reason kind of progress we are making
349 unindex_directory_scan_cb (void *cls,
350 const char *filename,
352 enum GNUNET_FS_DirScannerProgressUpdateReason reason)
354 struct GNUNET_FS_UnindexContext *uc = cls;
355 static struct GNUNET_FS_ShareTreeItem * directory_scan_result;
359 case GNUNET_FS_DIRSCANNER_FINISHED:
360 directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan);
362 if (NULL != directory_scan_result->ksk_uri)
364 uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri);
365 uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS;
366 GNUNET_FS_unindex_sync_ (uc);
367 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
371 uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan."));
372 GNUNET_FS_unindex_sync_ (uc);
375 GNUNET_FS_share_tree_free (directory_scan_result);
377 case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
378 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
379 _("Internal error scanning `%s'.\n"),
381 GNUNET_FS_directory_scan_abort (uc->dscan);
383 uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan."));
384 GNUNET_FS_unindex_sync_ (uc);
394 * If necessary, connect to the datastore and remove the UBlocks.
396 * @param uc context for the unindex operation.
399 GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc)
404 GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex))
406 uc->dscan = GNUNET_FS_directory_scan_start (uc->filename,
408 &unindex_directory_scan_cb,
410 GNUNET_free_non_null (ex);
415 * Continuation called to notify client about result of the remove
416 * operation for the UBlock.
418 * @param cls the 'struct GNUNET_FS_UnindexContext *'
419 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
420 * GNUNET_NO if content was already there
421 * GNUNET_YES (or other positive value) on success
422 * @param min_expiration minimum expiration time required for 0-priority content to be stored
423 * by the datacache at this time, zero for unknown, forever if we have no
424 * space for 0-priority content
425 * @param msg NULL on success, otherwise an error message
428 continue_after_remove (void *cls,
430 struct GNUNET_TIME_Absolute min_expiration,
433 struct GNUNET_FS_UnindexContext *uc = cls;
436 if (success != GNUNET_YES)
437 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
438 _("Failed to remove UBlock: %s\n"),
441 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
446 * Function called from datastore with result from us looking for
447 * a UBlock. There are four cases:
448 * 1) no result, means we move on to the next keyword
449 * 2) data hash is the same as an already seen data hash, means we move on to
451 * 3) UBlock for a different CHK, means we keep looking for more
452 * 4) UBlock is for our CHK, means we remove the block and then move
453 * on to the next keyword
455 * @param cls the 'struct GNUNET_FS_UnindexContext *'
456 * @param key key for the content
457 * @param size number of bytes in data
458 * @param data content stored
459 * @param type type of the content
460 * @param priority priority of the content
461 * @param anonymity anonymity-level for the content
462 * @param replication replication-level for the content
463 * @param expiration expiration time for the content
464 * @param uid unique identifier for the datum;
465 * maybe 0 if no unique identifier is available
468 process_kblock_for_unindex (void *cls,
469 const struct GNUNET_HashCode *key,
472 enum GNUNET_BLOCK_Type type,
475 uint32_t replication,
476 struct GNUNET_TIME_Absolute expiration,
479 struct GNUNET_FS_UnindexContext *uc = cls;
480 const struct UBlock *ub;
481 struct GNUNET_FS_Uri *chk_uri;
482 struct GNUNET_HashCode query;
489 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
492 GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
493 if (size < sizeof (struct UBlock))
499 GNUNET_CRYPTO_hash (&ub->verification_key,
500 sizeof (ub->verification_key),
502 if (0 != memcmp (&query,
504 sizeof (struct GNUNET_HashCode)))
506 /* result does not match our keyword, skip */
510 char pt[size - sizeof (struct UBlock)];
511 struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
514 GNUNET_CRYPTO_ecdsa_key_get_public (GNUNET_CRYPTO_ecdsa_key_get_anonymous (),
516 keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
517 GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof (struct UBlock),
521 if (NULL == memchr (&pt[1], 0, sizeof (pt) - 1))
523 GNUNET_break_op (0); /* malformed UBlock */
526 chk_uri = GNUNET_FS_uri_parse (&pt[1], NULL);
529 GNUNET_break_op (0); /* malformed UBlock */
533 if (0 != memcmp (&uc->chk,
534 &chk_uri->data.chk.chk,
535 sizeof (struct ContentHashKey)))
537 /* different CHK, ignore */
538 GNUNET_FS_uri_destroy (chk_uri);
541 GNUNET_FS_uri_destroy (chk_uri);
543 uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
549 &continue_after_remove,
553 uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
554 uid + 1 /* next_uid */,
557 GNUNET_BLOCK_TYPE_FS_UBLOCK,
560 &process_kblock_for_unindex,
566 * If necessary, connect to the datastore and remove the KBlocks.
568 * @param uc context for the unindex operation.
571 GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
574 const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
575 struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
576 struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
579 uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
582 uc->state = UNINDEX_STATE_ERROR;
583 uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service."));
584 GNUNET_FS_unindex_sync_ (uc);
585 signal_unindex_error (uc);
588 if ( (NULL == uc->ksk_uri) ||
589 (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) )
594 anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
595 GNUNET_CRYPTO_ecdsa_key_get_public (anon,
597 keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
598 GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub,
602 GNUNET_CRYPTO_hash (&dpub,
605 uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
609 GNUNET_BLOCK_TYPE_FS_UBLOCK,
612 &process_kblock_for_unindex,
618 * Function called when the tree encoder has
619 * processed all blocks. Clean up.
621 * @param cls our unindexing context
624 unindex_extract_keywords (void *cls)
626 struct GNUNET_FS_UnindexContext *uc = cls;
628 uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS;
629 GNUNET_FS_unindex_sync_ (uc);
630 GNUNET_FS_unindex_do_extract_keywords_ (uc);
635 * Connect to the datastore and remove the blocks.
637 * @param uc context for the unindex operation.
640 GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc)
643 uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
646 uc->state = UNINDEX_STATE_ERROR;
647 uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service."));
648 GNUNET_FS_unindex_sync_ (uc);
649 signal_unindex_error (uc);
653 GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ,
654 GNUNET_DISK_PERM_NONE);
657 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
659 uc->state = UNINDEX_STATE_ERROR;
660 uc->emsg = GNUNET_strdup (_("Failed to open file for unindexing."));
661 GNUNET_FS_unindex_sync_ (uc);
662 signal_unindex_error (uc);
666 GNUNET_FS_tree_encoder_create (uc->h,
672 &unindex_extract_keywords);
673 GNUNET_FS_tree_encoder_next (uc->tc);
678 * Function called once the hash of the file
679 * that is being unindexed has been computed.
681 * @param cls closure, unindex context
682 * @param file_id computed hash, NULL on error
685 GNUNET_FS_unindex_process_hash_ (void *cls,
686 const struct GNUNET_HashCode *file_id)
688 struct GNUNET_FS_UnindexContext *uc = cls;
691 if (uc->state != UNINDEX_STATE_HASHING)
693 GNUNET_FS_unindex_stop (uc);
698 uc->state = UNINDEX_STATE_ERROR;
699 uc->emsg = GNUNET_strdup (_("Failed to compute hash of file."));
700 GNUNET_FS_unindex_sync_ (uc);
701 signal_unindex_error (uc);
704 uc->file_id = *file_id;
705 uc->state = UNINDEX_STATE_DS_REMOVE;
706 GNUNET_FS_unindex_sync_ (uc);
707 GNUNET_FS_unindex_do_remove_ (uc);
712 * Create SUSPEND event for the given unindex operation
713 * and then clean up our state (without stop signal).
715 * @param cls the `struct GNUNET_FS_UnindexContext` to signal for
718 GNUNET_FS_unindex_signal_suspend_ (void *cls)
720 struct GNUNET_FS_UnindexContext *uc = cls;
721 struct GNUNET_FS_ProgressInfo pi;
723 /* FIXME: lots of duplication with unindex_stop here! */
724 if (uc->dscan != NULL)
726 GNUNET_FS_directory_scan_abort (uc->dscan);
731 GNUNET_DATASTORE_cancel (uc->dqe);
736 GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
739 if (NULL != uc->ksk_uri)
741 GNUNET_FS_uri_destroy (uc->ksk_uri);
746 GNUNET_MQ_destroy (uc->mq);
751 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
756 GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
761 GNUNET_DISK_file_close (uc->fh);
764 GNUNET_FS_end_top (uc->h, uc->top);
765 pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND;
766 GNUNET_FS_unindex_make_status_ (&pi, uc,
768 UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
769 GNUNET_break (NULL == uc->client_info);
770 GNUNET_free (uc->filename);
771 GNUNET_free_non_null (uc->serialization);
772 GNUNET_free_non_null (uc->emsg);
780 * @param h handle to the file sharing subsystem
781 * @param filename file to unindex
782 * @param cctx initial value for the client context
783 * @return NULL on error, otherwise handle
785 struct GNUNET_FS_UnindexContext *
786 GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
787 const char *filename,
790 struct GNUNET_FS_UnindexContext *uc;
791 struct GNUNET_FS_ProgressInfo pi;
795 GNUNET_DISK_file_size (filename,
800 uc = GNUNET_new (struct GNUNET_FS_UnindexContext);
802 uc->filename = GNUNET_strdup (filename);
803 uc->start_time = GNUNET_TIME_absolute_get ();
804 uc->file_size = size;
805 uc->client_info = cctx;
806 GNUNET_FS_unindex_sync_ (uc);
807 pi.status = GNUNET_FS_STATUS_UNINDEX_START;
808 pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
809 GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
811 GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
814 &GNUNET_FS_unindex_process_hash_, uc);
815 uc->top = GNUNET_FS_make_top (h,
816 &GNUNET_FS_unindex_signal_suspend_,
823 * Clean up after completion of an unindex operation.
828 GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc)
830 struct GNUNET_FS_ProgressInfo pi;
832 if (NULL != uc->dscan)
834 GNUNET_FS_directory_scan_abort (uc->dscan);
839 GNUNET_DATASTORE_cancel (uc->dqe);
844 GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
849 GNUNET_MQ_destroy (uc->mq);
854 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
857 if (NULL != uc->ksk_uri)
859 GNUNET_FS_uri_destroy (uc->ksk_uri);
864 GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
869 GNUNET_DISK_file_close (uc->fh);
872 GNUNET_FS_end_top (uc->h, uc->top);
873 if (uc->serialization != NULL)
875 GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
877 GNUNET_free (uc->serialization);
878 uc->serialization = NULL;
880 pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED;
881 pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
882 GNUNET_FS_unindex_make_status_ (&pi, uc,
884 UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
885 GNUNET_break (NULL == uc->client_info);
886 GNUNET_free_non_null (uc->emsg);
887 GNUNET_free (uc->filename);
891 /* end of fs_unindex.c */