2 This file is part of GNUnet.
3 Copyright (C) 2003--2013, 2016 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @file fs/fs_unindex.c
23 * @author Krista Grothoff
24 * @author Christian Grothoff
25 * @brief Unindex file.
28 #include "gnunet_constants.h"
29 #include "gnunet_fs_service.h"
30 #include "gnunet_protocols.h"
34 #include "fs_publish_ublock.h"
38 * Function called by the tree encoder to obtain
39 * a block of plaintext data (for the lowest level
42 * @param cls our publishing context
43 * @param offset identifies which block to get
44 * @param max (maximum) number of bytes to get; returning
45 * fewer will also cause errors
46 * @param buf where to copy the plaintext buffer
47 * @param emsg location to store an error message (on error)
48 * @return number of bytes copied to buf, 0 on error
51 unindex_reader (void *cls,
57 struct GNUNET_FS_UnindexContext *uc = cls;
60 pt_size = GNUNET_MIN (max, uc->file_size - offset);
61 if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET))
63 *emsg = GNUNET_strdup (_ ("Failed to find given position in file"));
66 if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size))
68 *emsg = GNUNET_strdup (_ ("Failed to read file"));
76 * Fill in all of the generic fields for
77 * an unindex event and call the callback.
79 * @param pi structure to fill in
80 * @param uc overall unindex context
81 * @param offset where we are in the file (for progress)
84 GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
85 struct GNUNET_FS_UnindexContext *uc,
88 pi->value.unindex.uc = uc;
89 pi->value.unindex.cctx = uc->client_info;
90 pi->value.unindex.filename = uc->filename;
91 pi->value.unindex.size = uc->file_size;
92 pi->value.unindex.eta =
93 GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size);
94 pi->value.unindex.duration =
95 GNUNET_TIME_absolute_get_duration (uc->start_time);
96 pi->value.unindex.completed = offset;
98 uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi);
103 * Function called with information about our
104 * progress in computing the tree encoding.
107 * @param offset where are we in the file
108 * @param pt_block plaintext of the currently processed block
109 * @param pt_size size of pt_block
110 * @param depth depth of the block in the tree, 0 for DBLOCK
113 unindex_progress (void *cls,
115 const void *pt_block,
119 struct GNUNET_FS_UnindexContext *uc = cls;
120 struct GNUNET_FS_ProgressInfo pi;
122 pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS;
123 pi.value.unindex.specifics.progress.data = pt_block;
124 pi.value.unindex.specifics.progress.offset = offset;
125 pi.value.unindex.specifics.progress.data_len = pt_size;
126 pi.value.unindex.specifics.progress.depth = depth;
127 GNUNET_FS_unindex_make_status_ (&pi, uc, offset);
132 * We've encountered an error during
133 * unindexing. Signal the client.
135 * @param uc context for the failed unindexing operation
138 signal_unindex_error (struct GNUNET_FS_UnindexContext *uc)
140 struct GNUNET_FS_ProgressInfo pi;
142 pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR;
143 pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
144 pi.value.unindex.specifics.error.message = uc->emsg;
145 GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
150 * Continuation called to notify client about result of the
151 * datastore removal operation.
154 * @param success #GNUNET_SYSERR on failure
155 * @param min_expiration minimum expiration time required for content to be stored
156 * @param msg NULL on success, otherwise an error message
159 process_cont (void *cls,
161 struct GNUNET_TIME_Absolute min_expiration,
164 struct GNUNET_FS_UnindexContext *uc = cls;
166 if (success == GNUNET_SYSERR)
168 uc->emsg = GNUNET_strdup (msg);
169 signal_unindex_error (uc);
172 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
173 "Datastore REMOVE operation succeeded\n");
174 GNUNET_FS_tree_encoder_next (uc->tc);
179 * Function called asking for the current (encoded)
180 * block to be processed. After processing the
181 * client should either call "GNUNET_FS_tree_encode_next"
182 * or (on error) "GNUNET_FS_tree_encode_finish".
185 * @param chk content hash key for the block (key for lookup in the datastore)
186 * @param offset offset of the block
187 * @param depth depth of the block, 0 for DBLOCK
188 * @param type type of the block (IBLOCK or DBLOCK)
189 * @param block the (encrypted) block
190 * @param block_size size of block (in bytes)
193 unindex_process (void *cls,
194 const struct ContentHashKey *chk,
197 enum GNUNET_BLOCK_Type type,
201 struct GNUNET_FS_UnindexContext *uc = cls;
204 struct OnDemandBlock odb;
206 if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK)
211 else /* on-demand encoded DBLOCK */
213 size = sizeof(struct OnDemandBlock);
214 odb.offset = GNUNET_htonll (offset);
215 odb.file_id = uc->file_id;
218 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
219 "Sending REMOVE request to DATASTORE service\n");
220 GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1,
227 * Function called with the response from the FS service to our
228 * unindexing request.
230 * @param cls closure, unindex context
231 * @param msg the response
234 handle_unindex_response (void *cls,
235 const struct GNUNET_MessageHeader *msg)
237 struct GNUNET_FS_UnindexContext *uc = cls;
238 struct GNUNET_FS_ProgressInfo pi;
242 GNUNET_MQ_destroy (uc->mq);
245 uc->state = UNINDEX_STATE_COMPLETE;
246 pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED;
247 pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
248 GNUNET_FS_unindex_sync_ (uc);
249 GNUNET_FS_unindex_make_status_ (&pi,
256 * Generic error handler, called with the appropriate error code and
257 * the same closure specified at the creation of the message queue.
258 * Not every message queue implementation supports an error handler.
260 * @param cls closure with the `struct GNUNET_FS_UnindexContext *`
261 * @param error error code
264 unindex_mq_error_handler (void *cls,
265 enum GNUNET_MQ_Error error)
267 struct GNUNET_FS_UnindexContext *uc = cls;
271 GNUNET_MQ_destroy (uc->mq);
274 uc->state = UNINDEX_STATE_ERROR;
275 uc->emsg = GNUNET_strdup (_ ("Error communicating with `fs' service."));
276 GNUNET_FS_unindex_sync_ (uc);
277 signal_unindex_error (uc);
282 * Function called when we are done with removing UBlocks.
283 * Disconnect from datastore and notify FS service about
286 * @param uc our unindexing context
289 unindex_finish (struct GNUNET_FS_UnindexContext *uc)
291 struct GNUNET_MQ_MessageHandler handlers[] = {
292 GNUNET_MQ_hd_fixed_size (unindex_response,
293 GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK,
294 struct GNUNET_MessageHeader,
296 GNUNET_MQ_handler_end ()
299 struct GNUNET_MQ_Envelope *env;
300 struct UnindexMessage *req;
302 /* generate final progress message */
303 unindex_progress (uc,
308 GNUNET_FS_tree_encoder_finish (uc->tc,
311 GNUNET_DISK_file_close (uc->fh);
313 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
315 uc->state = UNINDEX_STATE_FS_NOTIFY;
316 GNUNET_FS_unindex_sync_ (uc);
317 uc->mq = GNUNET_CLIENT_connect (uc->h->cfg,
320 &unindex_mq_error_handler,
324 uc->state = UNINDEX_STATE_ERROR;
326 GNUNET_strdup (_ ("Failed to connect to FS service for unindexing."));
327 GNUNET_FS_unindex_sync_ (uc);
328 signal_unindex_error (uc);
331 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
332 "Sending UNINDEX message to FS service\n");
333 env = GNUNET_MQ_msg (req,
334 GNUNET_MESSAGE_TYPE_FS_UNINDEX);
336 req->file_id = uc->file_id;
337 GNUNET_MQ_send (uc->mq,
343 * Function called by the directory scanner as we extract keywords
344 * that we will need to remove UBlocks.
346 * @param cls the 'struct GNUNET_FS_UnindexContext *'
347 * @param filename which file we are making progress on
348 * @param is_directory #GNUNET_YES if this is a directory,
349 * #GNUNET_NO if this is a file
350 * #GNUNET_SYSERR if it is neither (or unknown)
351 * @param reason kind of progress we are making
354 unindex_directory_scan_cb (void *cls,
355 const char *filename,
357 enum GNUNET_FS_DirScannerProgressUpdateReason reason)
359 struct GNUNET_FS_UnindexContext *uc = cls;
360 static struct GNUNET_FS_ShareTreeItem *directory_scan_result;
364 case GNUNET_FS_DIRSCANNER_FINISHED:
365 directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan);
367 if (NULL != directory_scan_result->ksk_uri)
369 uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri);
370 uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS;
371 GNUNET_FS_unindex_sync_ (uc);
372 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
376 uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan."));
377 GNUNET_FS_unindex_sync_ (uc);
380 GNUNET_FS_share_tree_free (directory_scan_result);
383 case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
384 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
385 _ ("Internal error scanning `%s'.\n"),
387 GNUNET_FS_directory_scan_abort (uc->dscan);
389 uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan."));
390 GNUNET_FS_unindex_sync_ (uc);
401 * If necessary, connect to the datastore and remove the UBlocks.
403 * @param uc context for the unindex operation.
406 GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc)
411 GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS",
414 uc->dscan = GNUNET_FS_directory_scan_start (uc->filename,
416 &unindex_directory_scan_cb,
418 GNUNET_free_non_null (ex);
423 * Continuation called to notify client about result of the remove
424 * operation for the UBlock.
426 * @param cls the 'struct GNUNET_FS_UnindexContext *'
427 * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
428 * GNUNET_NO if content was already there
429 * GNUNET_YES (or other positive value) on success
430 * @param min_expiration minimum expiration time required for 0-priority content to be stored
431 * by the datacache at this time, zero for unknown, forever if we have no
432 * space for 0-priority content
433 * @param msg NULL on success, otherwise an error message
436 continue_after_remove (void *cls,
438 struct GNUNET_TIME_Absolute min_expiration,
441 struct GNUNET_FS_UnindexContext *uc = cls;
444 if (success != GNUNET_YES)
445 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
446 _ ("Failed to remove UBlock: %s\n"),
449 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
454 * Function called from datastore with result from us looking for
455 * a UBlock. There are four cases:
456 * 1) no result, means we move on to the next keyword
457 * 2) data hash is the same as an already seen data hash, means we move on to
459 * 3) UBlock for a different CHK, means we keep looking for more
460 * 4) UBlock is for our CHK, means we remove the block and then move
461 * on to the next keyword
463 * @param cls the 'struct GNUNET_FS_UnindexContext *'
464 * @param key key for the content
465 * @param size number of bytes in data
466 * @param data content stored
467 * @param type type of the content
468 * @param priority priority of the content
469 * @param anonymity anonymity-level for the content
470 * @param replication replication-level for the content
471 * @param expiration expiration time for the content
472 * @param uid unique identifier for the datum;
473 * maybe 0 if no unique identifier is available
476 process_kblock_for_unindex (void *cls,
477 const struct GNUNET_HashCode *key,
480 enum GNUNET_BLOCK_Type type,
483 uint32_t replication,
484 struct GNUNET_TIME_Absolute expiration,
487 struct GNUNET_FS_UnindexContext *uc = cls;
488 const struct UBlock *ub;
489 struct GNUNET_FS_Uri *chk_uri;
490 struct GNUNET_HashCode query;
497 GNUNET_FS_unindex_do_remove_kblocks_ (uc);
500 GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
501 if (size < sizeof(struct UBlock))
507 GNUNET_CRYPTO_hash (&ub->verification_key,
508 sizeof(ub->verification_key),
510 if (0 != memcmp (&query,
512 sizeof(struct GNUNET_HashCode)))
514 /* result does not match our keyword, skip */
518 char pt[size - sizeof(struct UBlock)];
519 struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
522 GNUNET_CRYPTO_ecdsa_key_get_public (
523 GNUNET_CRYPTO_ecdsa_key_get_anonymous (),
525 keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
526 GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof(struct UBlock),
530 if (NULL == memchr (&pt[1], 0, sizeof(pt) - 1))
532 GNUNET_break_op (0); /* malformed UBlock */
535 chk_uri = GNUNET_FS_uri_parse (&pt[1], NULL);
538 GNUNET_break_op (0); /* malformed UBlock */
542 if (0 != memcmp (&uc->chk,
543 &chk_uri->data.chk.chk,
544 sizeof(struct ContentHashKey)))
546 /* different CHK, ignore */
547 GNUNET_FS_uri_destroy (chk_uri);
550 GNUNET_FS_uri_destroy (chk_uri);
552 uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
558 &continue_after_remove,
562 uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
563 uid + 1 /* next_uid */,
566 GNUNET_BLOCK_TYPE_FS_UBLOCK,
569 &process_kblock_for_unindex,
575 * If necessary, connect to the datastore and remove the KBlocks.
577 * @param uc context for the unindex operation.
580 GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
583 const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
584 struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
585 struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
588 uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
591 uc->state = UNINDEX_STATE_ERROR;
592 uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service."));
593 GNUNET_FS_unindex_sync_ (uc);
594 signal_unindex_error (uc);
597 if ((NULL == uc->ksk_uri) ||
598 (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount))
603 anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
604 GNUNET_CRYPTO_ecdsa_key_get_public (anon,
606 keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
607 GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub,
611 GNUNET_CRYPTO_hash (&dpub,
614 uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
618 GNUNET_BLOCK_TYPE_FS_UBLOCK,
621 &process_kblock_for_unindex,
627 * Function called when the tree encoder has
628 * processed all blocks. Clean up.
630 * @param cls our unindexing context
633 unindex_extract_keywords (void *cls)
635 struct GNUNET_FS_UnindexContext *uc = cls;
637 uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS;
638 GNUNET_FS_unindex_sync_ (uc);
639 GNUNET_FS_unindex_do_extract_keywords_ (uc);
644 * Connect to the datastore and remove the blocks.
646 * @param uc context for the unindex operation.
649 GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc)
652 uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
655 uc->state = UNINDEX_STATE_ERROR;
656 uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service."));
657 GNUNET_FS_unindex_sync_ (uc);
658 signal_unindex_error (uc);
662 GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ,
663 GNUNET_DISK_PERM_NONE);
666 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
668 uc->state = UNINDEX_STATE_ERROR;
669 uc->emsg = GNUNET_strdup (_ ("Failed to open file for unindexing."));
670 GNUNET_FS_unindex_sync_ (uc);
671 signal_unindex_error (uc);
675 GNUNET_FS_tree_encoder_create (uc->h,
681 &unindex_extract_keywords);
682 GNUNET_FS_tree_encoder_next (uc->tc);
687 * Function called once the hash of the file
688 * that is being unindexed has been computed.
690 * @param cls closure, unindex context
691 * @param file_id computed hash, NULL on error
694 GNUNET_FS_unindex_process_hash_ (void *cls,
695 const struct GNUNET_HashCode *file_id)
697 struct GNUNET_FS_UnindexContext *uc = cls;
700 if (uc->state != UNINDEX_STATE_HASHING)
702 GNUNET_FS_unindex_stop (uc);
707 uc->state = UNINDEX_STATE_ERROR;
708 uc->emsg = GNUNET_strdup (_ ("Failed to compute hash of file."));
709 GNUNET_FS_unindex_sync_ (uc);
710 signal_unindex_error (uc);
713 uc->file_id = *file_id;
714 uc->state = UNINDEX_STATE_DS_REMOVE;
715 GNUNET_FS_unindex_sync_ (uc);
716 GNUNET_FS_unindex_do_remove_ (uc);
721 * Create SUSPEND event for the given unindex operation
722 * and then clean up our state (without stop signal).
724 * @param cls the `struct GNUNET_FS_UnindexContext` to signal for
727 GNUNET_FS_unindex_signal_suspend_ (void *cls)
729 struct GNUNET_FS_UnindexContext *uc = cls;
730 struct GNUNET_FS_ProgressInfo pi;
732 /* FIXME: lots of duplication with unindex_stop here! */
733 if (uc->dscan != NULL)
735 GNUNET_FS_directory_scan_abort (uc->dscan);
740 GNUNET_DATASTORE_cancel (uc->dqe);
745 GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
748 if (NULL != uc->ksk_uri)
750 GNUNET_FS_uri_destroy (uc->ksk_uri);
755 GNUNET_MQ_destroy (uc->mq);
760 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
765 GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
770 GNUNET_DISK_file_close (uc->fh);
773 GNUNET_FS_end_top (uc->h, uc->top);
774 pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND;
775 GNUNET_FS_unindex_make_status_ (&pi, uc,
777 UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
778 GNUNET_break (NULL == uc->client_info);
779 GNUNET_free (uc->filename);
780 GNUNET_free_non_null (uc->serialization);
781 GNUNET_free_non_null (uc->emsg);
789 * @param h handle to the file sharing subsystem
790 * @param filename file to unindex
791 * @param cctx initial value for the client context
792 * @return NULL on error, otherwise handle
794 struct GNUNET_FS_UnindexContext *
795 GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
796 const char *filename,
799 struct GNUNET_FS_UnindexContext *uc;
800 struct GNUNET_FS_ProgressInfo pi;
804 GNUNET_DISK_file_size (filename,
809 uc = GNUNET_new (struct GNUNET_FS_UnindexContext);
811 uc->filename = GNUNET_strdup (filename);
812 uc->start_time = GNUNET_TIME_absolute_get ();
813 uc->file_size = size;
814 uc->client_info = cctx;
815 GNUNET_FS_unindex_sync_ (uc);
816 pi.status = GNUNET_FS_STATUS_UNINDEX_START;
817 pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
818 GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
820 GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
823 &GNUNET_FS_unindex_process_hash_, uc);
824 uc->top = GNUNET_FS_make_top (h,
825 &GNUNET_FS_unindex_signal_suspend_,
832 * Clean up after completion of an unindex operation.
837 GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc)
839 struct GNUNET_FS_ProgressInfo pi;
841 if (NULL != uc->dscan)
843 GNUNET_FS_directory_scan_abort (uc->dscan);
848 GNUNET_DATASTORE_cancel (uc->dqe);
853 GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
858 GNUNET_MQ_destroy (uc->mq);
863 GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
866 if (NULL != uc->ksk_uri)
868 GNUNET_FS_uri_destroy (uc->ksk_uri);
873 GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
878 GNUNET_DISK_file_close (uc->fh);
881 GNUNET_FS_end_top (uc->h, uc->top);
882 if (uc->serialization != NULL)
884 GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
886 GNUNET_free (uc->serialization);
887 uc->serialization = NULL;
889 pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED;
890 pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
891 GNUNET_FS_unindex_make_status_ (&pi, uc,
893 UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
894 GNUNET_break (NULL == uc->client_info);
895 GNUNET_free_non_null (uc->emsg);
896 GNUNET_free (uc->filename);
901 /* end of fs_unindex.c */