X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Ffs%2Ffs_unindex.c;h=71a1b9e0ba78c4195904693d9c3a113801e19091;hb=eef168bdbf40609b944dada415fc536e15cac494;hp=c32a10cbf862451780aac4dfda4b9cc4594f98df;hpb=0634957156088bc3c53d787c13ba3006307d8ca5;p=oweals%2Fgnunet.git diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index c32a10cbf..71a1b9e0b 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -4,7 +4,7 @@ GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your + by the Free Software Foundation; either version 3, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but @@ -20,621 +20,830 @@ /** * @file fs/fs_unindex.c - * @author Krista Bennett + * @author Krista Grothoff * @author Christian Grothoff * @brief Unindex file. - * - * TODO: - * - code cleanup (share more with upload.c) */ - #include "platform.h" #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" +#include "fs_tree.h" +#include "block_fs.h" /** - * Context for "GNUNET_FS_get_indexed_files". + * Function called by the tree encoder to obtain + * a block of plaintext data (for the lowest level + * of the tree). + * + * @param cls our publishing context + * @param offset identifies which block to get + * @param max (maximum) number of bytes to get; returning + * fewer will also cause errors + * @param buf where to copy the plaintext buffer + * @param emsg location to store an error message (on error) + * @return number of bytes copied to buf, 0 on error */ -struct GetIndexedContext +static size_t +unindex_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) { - /** - * Handle to global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Connection to the FS service. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Function to call for each indexed file. - */ - GNUNET_FS_IndexedFileProcessor iterator; - - /** - * Closure for iterator. - */ - void *iterator_cls; - - /** - * Continuation to trigger at the end. - */ - GNUNET_SCHEDULER_Task cont; - - /** - * Closure for cont. - */ - void *cont_cls; -}; + struct GNUNET_FS_UnindexContext *uc = cls; + size_t pt_size; + + pt_size = GNUNET_MIN (max, uc->file_size - offset); + if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET)) + { + *emsg = GNUNET_strdup (_("Failed to find given position in file")); + return 0; + } + if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size)) + { + *emsg = GNUNET_strdup (_("Failed to read file")); + return 0; + } + return pt_size; +} /** - * Function called on each response from the FS - * service with information about indexed files. + * Fill in all of the generic fields for + * an unindex event and call the callback. * - * @param cls closure (of type "struct GetIndexedContext*") - * @param msg message with indexing information + * @param pi structure to fill in + * @param uc overall unindex context + * @param offset where we are in the file (for progress) */ -static void -handle_index_info (void *cls, - const struct GNUNET_MessageHeader *msg) +void +GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_UnindexContext *uc, + uint64_t offset) { - struct GetIndexedContext *gic = cls; - const struct IndexInfoMessage *iim; - uint16_t msize; - const char *filename; + pi->value.unindex.uc = uc; + pi->value.unindex.cctx = uc->client_info; + pi->value.unindex.filename = uc->filename; + pi->value.unindex.size = uc->file_size; + pi->value.unindex.eta = + GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size); + pi->value.unindex.duration = + GNUNET_TIME_absolute_get_duration (uc->start_time); + pi->value.unindex.completed = offset; + uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi); - if (NULL == msg) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to receive response for `%s' request from `%s' service.\n"), - "GET_INDEXED", - "fs"); - GNUNET_SCHEDULER_add_continuation (gic->h->sched, - GNUNET_NO, - gic->cont, - gic->cont_cls, - GNUNET_SCHEDULER_REASON_TIMEOUT); - GNUNET_CLIENT_disconnect (gic->client); - GNUNET_free (gic); - return; - } - if (ntohs (msg->type) == GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END) - { - /* normal end-of-list */ - GNUNET_SCHEDULER_add_continuation (gic->h->sched, - GNUNET_NO, - gic->cont, - gic->cont_cls, - GNUNET_SCHEDULER_REASON_PREREQ_DONE); - GNUNET_CLIENT_disconnect (gic->client); - GNUNET_free (gic); - return; - } - msize = ntohs (msg->size); - iim = (const struct IndexInfoMessage*) msg; - filename = (const char*) &iim[1]; - if ( (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY) || - (msize <= sizeof (struct IndexInfoMessage)) || - (filename[msize-sizeof (struct IndexInfoMessage) -1] != '\0') ) - { - /* bogus reply */ - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to receive valid response for `%s' request from `%s' service.\n"), - "GET_INDEXED", - "fs"); - GNUNET_SCHEDULER_add_continuation (gic->h->sched, - GNUNET_NO, - gic->cont, - gic->cont_cls, - GNUNET_SCHEDULER_REASON_TIMEOUT); - GNUNET_CLIENT_disconnect (gic->client); - GNUNET_free (gic); - return; - } - if (GNUNET_OK != - gic->iterator (gic->iterator_cls, - filename, - &iim->file_id)) - { - GNUNET_SCHEDULER_add_continuation (gic->h->sched, - GNUNET_NO, - gic->cont, - gic->cont_cls, - GNUNET_SCHEDULER_REASON_PREREQ_DONE); - GNUNET_CLIENT_disconnect (gic->client); - GNUNET_free (gic); - return; - } - /* get more */ - GNUNET_CLIENT_receive (gic->client, - &handle_index_info, - gic, - GNUNET_CONSTANTS_SERVICE_TIMEOUT); } /** - * Transmit the request to get a list of all - * indexed files to the "FS" service. + * Function called with information about our + * progress in computing the tree encoding. * - * @param cls closure (of type "struct GetIndexedContext*") - * @param size number of bytes availabe in buf - * @param buf where to write the message, NULL on error - * @return number of bytes written to buf + * @param cls closure + * @param offset where are we in the file + * @param pt_block plaintext of the currently processed block + * @param pt_size size of pt_block + * @param depth depth of the block in the tree, 0 for DBLOCK */ -static size_t -transmit_get_indexed (void *cls, - size_t size, - void *buf) +static void +unindex_progress (void *cls, uint64_t offset, const void *pt_block, + size_t pt_size, unsigned int depth) { - struct GetIndexedContext *gic = cls; - struct GNUNET_MessageHeader *hdr; - - if (NULL == buf) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to transmit `%s' request to `%s' service.\n"), - "GET_INDEXED", - "fs"); - GNUNET_SCHEDULER_add_continuation (gic->h->sched, - GNUNET_NO, - gic->cont, - gic->cont_cls, - GNUNET_SCHEDULER_REASON_TIMEOUT); - GNUNET_CLIENT_disconnect (gic->client); - GNUNET_free (gic); - return 0; - } - GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader)); - hdr = buf; - hdr->size = htons (sizeof (struct GNUNET_MessageHeader)); - hdr->type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET); - GNUNET_CLIENT_receive (gic->client, - &handle_index_info, - gic, - GNUNET_CONSTANTS_SERVICE_TIMEOUT); - return sizeof (struct GNUNET_MessageHeader); + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS; + pi.value.unindex.specifics.progress.data = pt_block; + pi.value.unindex.specifics.progress.offset = offset; + pi.value.unindex.specifics.progress.data_len = pt_size; + pi.value.unindex.specifics.progress.depth = depth; + GNUNET_FS_unindex_make_status_ (&pi, uc, offset); } /** - * Iterate over all indexed files. + * We've encountered an error during + * unindexing. Signal the client. * - * @param h handle to the file sharing subsystem - * @param iterator function to call on each indexed file - * @param iterator_cls closure for iterator - * @param cont continuation to call when done; - * reason should be "TIMEOUT" (on - * error) or "PREREQ_DONE" (on success) - * @param cont_cls closure for cont + * @param uc context for the failed unindexing operation */ -void -GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h, - GNUNET_FS_IndexedFileProcessor iterator, - void *iterator_cls, - GNUNET_SCHEDULER_Task cont, - void *cont_cls) +static void +signal_unindex_error (struct GNUNET_FS_UnindexContext *uc) { - struct GNUNET_CLIENT_Connection *client; - struct GetIndexedContext *gic; - - client = GNUNET_CLIENT_connect (h->sched, - "fs", - h->cfg); - if (NULL == client) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to not connect to `%s' service.\n"), - "fs"); - GNUNET_SCHEDULER_add_continuation (h->sched, - GNUNET_NO, - cont, - cont_cls, - GNUNET_SCHEDULER_REASON_TIMEOUT); - return; - } + struct GNUNET_FS_ProgressInfo pi; - gic = GNUNET_malloc (sizeof (struct GetIndexedContext)); - gic->h = h; - gic->client = client; - gic->iterator = iterator; - gic->iterator_cls = iterator_cls; - gic->cont = cont; - gic->cont_cls = cont_cls; - GNUNET_CLIENT_notify_transmit_ready (client, - sizeof (struct GNUNET_MessageHeader), - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &transmit_get_indexed, - gic); + pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR; + pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.unindex.specifics.error.message = uc->emsg; + GNUNET_FS_unindex_make_status_ (&pi, uc, 0); } /** - * Unindex a file. + * Continuation called to notify client about result of the + * datastore removal operation. * - * @param h handle to the file sharing subsystem - * @param filename file to unindex - * @return NULL on error, otherwise handle + * @param cls closure + * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored + * @param msg NULL on success, otherwise an error message */ -struct GNUNET_FS_UnindexContext * -GNUNET_FS_unindex (struct GNUNET_FS_Handle *h, - const char *filename) +static void +process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) { - return NULL; + struct GNUNET_FS_UnindexContext *uc = cls; + + if (success == GNUNET_SYSERR) + { + uc->emsg = GNUNET_strdup (msg); + signal_unindex_error (uc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Datastore REMOVE operation succeeded\n"); + GNUNET_FS_tree_encoder_next (uc->tc); } /** - * Clean up after completion of an unindex operation. + * Function called asking for the current (encoded) + * block to be processed. After processing the + * client should either call "GNUNET_FS_tree_encode_next" + * or (on error) "GNUNET_FS_tree_encode_finish". * - * @param uc handle + * @param cls closure + * @param chk content hash key for the block (key for lookup in the datastore) + * @param offset offset of the block + * @param depth depth of the block, 0 for DBLOCK + * @param type type of the block (IBLOCK or DBLOCK) + * @param block the (encrypted) block + * @param block_size size of block (in bytes) */ -void -GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) +static void +unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, + unsigned int depth, enum GNUNET_BLOCK_Type type, + const void *block, uint16_t block_size) { + struct GNUNET_FS_UnindexContext *uc = cls; + uint32_t size; + const void *data; + struct OnDemandBlock odb; + + if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK) + { + size = block_size; + data = block; + } + else /* on-demand encoded DBLOCK */ + { + size = sizeof (struct OnDemandBlock); + odb.offset = GNUNET_htonll (offset); + odb.file_id = uc->file_id; + data = &odb; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Sending REMOVE request to DATASTORE service\n"); + GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); + uc->chk = *chk; } +/** + * Function called with the response from the + * FS service to our unindexing request. + * + * @param cls closure, unindex context + * @param msg NULL on timeout, otherwise the response + */ +static void +process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client); + uc->client = NULL; + } + if (uc->state != UNINDEX_STATE_FS_NOTIFY) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = + GNUNET_strdup (_("Unexpected time for a response from `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if (NULL == msg) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Timeout waiting for `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Invalid response from `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->state = UNINDEX_STATE_COMPLETE; + pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED; + pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_make_status_ (&pi, uc, uc->file_size); +} -#if 0 - -#define STRICT_CHECKS GNUNET_NO /** - * Append the given key and query to the iblock[level]. - * If iblock[level] is already full, compute its chk - * and push it to level+1. iblocks is guaranteed to - * be big enough. + * Function called when we are done with removing KBlocks. + * Disconnect from datastore and notify FS service about + * the unindex event. * - * This function matches exactly upload.c::pushBlock, - * except in the call to 'GNUNET_FS_delete'. TODO: refactor - * to avoid code duplication (move to block.c, pass - * GNUNET_FS_delete as argument!). + * @param uc our unindexing context */ -static int -pushBlock (struct GNUNET_ClientServerConnection *sock, - const GNUNET_EC_ContentHashKey * chk, unsigned int level, - GNUNET_DatastoreValue ** iblocks) +static void +unindex_finish (struct GNUNET_FS_UnindexContext *uc) { - unsigned int size; - unsigned int present; - GNUNET_DatastoreValue *value; - GNUNET_EC_DBlock *db; - GNUNET_EC_ContentHashKey ichk; - - size = ntohl (iblocks[level]->size) - sizeof (GNUNET_DatastoreValue); - present = - (size - sizeof (GNUNET_EC_DBlock)) / sizeof (GNUNET_EC_ContentHashKey); - db = (GNUNET_EC_DBlock *) & iblocks[level][1]; - if (present == GNUNET_ECRS_CHK_PER_INODE) - { - GNUNET_EC_file_block_get_key (db, size, &ichk.key); - GNUNET_EC_file_block_get_query (db, size, &ichk.query); - if (GNUNET_OK != pushBlock (sock, &ichk, level + 1, iblocks)) - { - GNUNET_GE_BREAK (NULL, 0); - return GNUNET_SYSERR; - } - GNUNET_EC_file_block_encode (db, size, &ichk.query, &value); -#if STRICT_CHECKS - if (GNUNET_SYSERR == GNUNET_FS_delete (sock, value)) - { - GNUNET_free (value); - GNUNET_GE_BREAK (NULL, 0); - return GNUNET_SYSERR; - } -#else - GNUNET_FS_delete (sock, value); -#endif - GNUNET_free (value); - size = sizeof (GNUNET_EC_DBlock); - } - /* append GNUNET_EC_ContentHashKey */ - memcpy (&((char *) db)[size], chk, sizeof (GNUNET_EC_ContentHashKey)); - iblocks[level]->size = htonl (size + - sizeof (GNUNET_EC_ContentHashKey) + - sizeof (GNUNET_DatastoreValue)); - return GNUNET_OK; + char *emsg; + struct GNUNET_FS_Uri *uri; + struct UnindexMessage req; + + /* generate final progress message */ + unindex_progress (uc, uc->file_size, NULL, 0, 0); + GNUNET_FS_tree_encoder_finish (uc->tc, &uri, &emsg); + uc->tc = NULL; + if (uri != NULL) + GNUNET_FS_uri_destroy (uri); + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + uc->state = UNINDEX_STATE_FS_NOTIFY; + GNUNET_FS_unindex_sync_ (uc); + uc->client = GNUNET_CLIENT_connect ("fs", uc->h->cfg); + if (uc->client == NULL) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = + GNUNET_strdup (_("Failed to connect to FS service for unindexing.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Sending UNINDEX message to FS service\n"); + req.header.size = htons (sizeof (struct UnindexMessage)); + req.header.type = htons (GNUNET_MESSAGE_TYPE_FS_UNINDEX); + req.reserved = 0; + req.file_id = uc->file_id; + GNUNET_break (GNUNET_OK == + GNUNET_CLIENT_transmit_and_get_response (uc->client, + &req.header, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_YES, + &process_fs_response, + uc)); } /** - * Undo sym-linking operation: - * a) check if we have a symlink - * b) delete symbolic link + * Function called by the directory scanner as we extract keywords + * that we will need to remove KBlocks. + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param filename which file we are making progress on + * @param is_directory GNUNET_YES if this is a directory, + * GNUNET_NO if this is a file + * GNUNET_SYSERR if it is neither (or unknown) + * @param reason kind of progress we are making */ -static int -undoSymlinking (struct GNUNET_GE_Context *ectx, - const char *fn, - const GNUNET_HashCode * fileId, - struct GNUNET_ClientServerConnection *sock) +static void +unindex_directory_scan_cb (void *cls, + const char *filename, + int is_directory, + enum GNUNET_FS_DirScannerProgressUpdateReason reason) { - GNUNET_EncName enc; - char *serverDir; - char *serverFN; - struct stat buf; - -#ifndef S_ISLNK - if (1) - return GNUNET_OK; /* symlinks do not exist? */ -#endif - if (0 != LSTAT (fn, &buf)) + struct GNUNET_FS_UnindexContext *uc = cls; + static struct GNUNET_FS_ShareTreeItem * directory_scan_result; + + switch (reason) + { + case GNUNET_FS_DIRSCANNER_FINISHED: + directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan); + uc->dscan = NULL; + if (NULL != directory_scan_result->ksk_uri) { - GNUNET_GE_LOG_STRERROR_FILE (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | - GNUNET_GE_USER | GNUNET_GE_ADMIN, "stat", - fn); - return GNUNET_SYSERR; + uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri); + uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS; + uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan.")); + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_remove_kblocks_ (uc); } -#ifdef S_ISLNK - if (!S_ISLNK (buf.st_mode)) - return GNUNET_OK; -#endif - serverDir = - GNUNET_get_daemon_configuration_value (sock, "FS", "INDEX-DIRECTORY"); - if (serverDir == NULL) - return GNUNET_OK; - serverFN = GNUNET_malloc (strlen (serverDir) + 2 + sizeof (GNUNET_EncName)); - strcpy (serverFN, serverDir); - GNUNET_free (serverDir); - if (serverFN[strlen (serverFN) - 1] != DIR_SEPARATOR) - strcat (serverFN, DIR_SEPARATOR_STR); - GNUNET_hash_to_enc (fileId, &enc); - strcat (serverFN, (char *) &enc); - - if (0 != UNLINK (serverFN)) + else { - GNUNET_GE_LOG_STRERROR_FILE (ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | - GNUNET_GE_USER | GNUNET_GE_ADMIN, "unlink", - serverFN); - GNUNET_free (serverFN); - return GNUNET_SYSERR; + unindex_finish (uc); } - GNUNET_free (serverFN); - return GNUNET_OK; -} + GNUNET_FS_share_tree_free (directory_scan_result); + break; + case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Internal error scanning `%s'.\n"), + uc->filename); + break; + default: + break; + } +} /** - * Unindex a file. + * If necessary, connect to the datastore and remove the KBlocks. * - * @return GNUNET_SYSERR if the unindexing failed (i.e. not indexed) + * @param uc context for the unindex operation. */ -int -GNUNET_ECRS_file_unindex (struct GNUNET_GE_Context *ectx, - struct GNUNET_GC_Configuration *cfg, - const char *filename, - GNUNET_ECRS_UploadProgressCallback upcb, - void *upcbClosure, GNUNET_ECRS_TestTerminate tt, - void *ttClosure) +void +GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc) { - unsigned long long filesize; - unsigned long long pos; - unsigned int treedepth; - int fd; - int i; - unsigned int size; - GNUNET_DatastoreValue **iblocks; - GNUNET_DatastoreValue *dblock; - GNUNET_EC_DBlock *db; - GNUNET_DatastoreValue *value; - struct GNUNET_ClientServerConnection *sock; - GNUNET_HashCode fileId; - GNUNET_EC_ContentHashKey chk; - GNUNET_CronTime eta; - GNUNET_CronTime start; - GNUNET_CronTime now; - int wasIndexed; - - start = GNUNET_get_time (); - if (GNUNET_YES != GNUNET_disk_file_test (ectx, filename)) - { - GNUNET_GE_BREAK (ectx, 0); - return GNUNET_SYSERR; - } + char *ex; + if (GNUNET_OK != - GNUNET_disk_file_size (ectx, filename, &filesize, GNUNET_YES)) - return GNUNET_SYSERR; - sock = GNUNET_client_connection_create (ectx, cfg); - if (sock == NULL) - return GNUNET_SYSERR; - eta = 0; - if (upcb != NULL) - upcb (filesize, 0, eta, upcbClosure); - if (GNUNET_SYSERR == GNUNET_hash_file (ectx, filename, &fileId)) + GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex)) + ex = NULL; + uc->dscan = GNUNET_FS_directory_scan_start (uc->filename, + GNUNET_NO, ex, + &unindex_directory_scan_cb, + uc); + GNUNET_free_non_null (ex); +} + + +/** + * Continuation called to notify client about result of the remove + * operation for the KBlock. + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param success GNUNET_SYSERR on failure (including timeout/queue drop) + * GNUNET_NO if content was already there + * GNUNET_YES (or other positive value) on success + * @param min_expiration minimum expiration time required for 0-priority content to be stored + * by the datacache at this time, zero for unknown, forever if we have no + * space for 0-priority content + * @param msg NULL on success, otherwise an error message + */ +static void +continue_after_remove (void *cls, + int32_t success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->dqe = NULL; + if (success != GNUNET_YES) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to remove KBlock: %s\n"), + msg); + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); +} + + +/** + * Function called from datastore with result from us looking for + * a KBlock. There are four cases: + * 1) no result, means we move on to the next keyword + * 2) UID is the same as the first UID, means we move on to next keyword + * 3) KBlock for a different CHK, means we keep looking for more + * 4) KBlock is for our CHK, means we remove the block and then move + * on to the next keyword + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +process_kblock_for_unindex (void *cls, + const struct GNUNET_HashCode * key, + size_t size, const void *data, + enum GNUNET_BLOCK_Type type, + uint32_t priority, + uint32_t anonymity, + struct GNUNET_TIME_Absolute + expiration, uint64_t uid) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + const struct KBlock *kb; + struct GNUNET_FS_Uri *chk_uri; + + uc->dqe = NULL; + if (NULL == data) + { + /* no result */ + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); + return; + } + if (0 == uc->first_uid) + { + /* remember UID of first result to detect cycles */ + uc->first_uid = uid; + } + else if (uid == uc->first_uid) + { + /* no more additional results */ + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); + return; + } + GNUNET_assert (GNUNET_BLOCK_TYPE_FS_KBLOCK == type); + if (size < sizeof (struct KBlock)) + { + GNUNET_break (0); + goto get_next; + } + kb = data; + { + char pt[size - sizeof (struct KBlock)]; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + + GNUNET_CRYPTO_hash_to_aes_key (&uc->key, &skey, &iv); + if (-1 == + GNUNET_CRYPTO_aes_decrypt (&kb[1], size - sizeof (struct KBlock), &skey, + &iv, pt)) { - GNUNET_client_connection_destroy (sock); - GNUNET_GE_BREAK (ectx, 0); - return GNUNET_SYSERR; - } - now = GNUNET_get_time (); - eta = now + 2 * (now - start); - /* very rough estimate: GNUNET_hash reads once through the file, - we'll do that once more and write it. But of course - the second read may be cached, and we have the encryption, - so a factor of two is really, really just a rough estimate */ - start = now; - /* reset the counter since the formula later does not - take the time for GNUNET_hash_file into account */ - treedepth = GNUNET_ECRS_compute_depth (filesize); - - /* Test if file is indexed! */ - wasIndexed = GNUNET_FS_test_indexed (sock, &fileId); - - fd = GNUNET_disk_file_open (ectx, filename, O_RDONLY | O_LARGEFILE); - if (fd == -1) + GNUNET_break (0); + goto get_next; + } + if (NULL == memchr (pt, 0, sizeof (pt))) { - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; + GNUNET_break (0); + goto get_next; } - dblock = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE + - sizeof (GNUNET_EC_DBlock)); - dblock->size = - htonl (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE + - sizeof (GNUNET_EC_DBlock)); - dblock->anonymity_level = htonl (0); - dblock->priority = htonl (0); - dblock->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - dblock->expiration_time = GNUNET_htonll (0); - db = (GNUNET_EC_DBlock *) & dblock[1]; - db->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - iblocks = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue *) * (treedepth + 1)); - for (i = 0; i <= treedepth; i++) + chk_uri = GNUNET_FS_uri_parse (pt, NULL); + if (NULL == chk_uri) { - iblocks[i] = - GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + - GNUNET_ECRS_IBLOCK_SIZE + sizeof (GNUNET_EC_DBlock)); - iblocks[i]->size = - htonl (sizeof (GNUNET_DatastoreValue) + sizeof (GNUNET_EC_DBlock)); - iblocks[i]->anonymity_level = htonl (0); - iblocks[i]->priority = htonl (0); - iblocks[i]->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA); - iblocks[i]->expiration_time = GNUNET_htonll (0); - ((GNUNET_EC_DBlock *) & iblocks[i][1])->type = - htonl (GNUNET_ECRS_BLOCKTYPE_DATA); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to parse URI `%s' from KBlock!\n"), + pt); + GNUNET_break (0); + goto get_next; } + } + if (0 != memcmp (&uc->chk, + &chk_uri->data.chk.chk, + sizeof (struct ContentHashKey))) + { + /* different CHK, ignore */ + GNUNET_FS_uri_destroy (chk_uri); + goto get_next; + } + GNUNET_FS_uri_destroy (chk_uri); + /* matches! */ + uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, + key, size, data, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &continue_after_remove, + uc); + return; + get_next: + uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, + uc->roff++, + &uc->query, + GNUNET_BLOCK_TYPE_FS_KBLOCK, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &process_kblock_for_unindex, + uc); +} - pos = 0; - while (pos < filesize) - { - if (upcb != NULL) - upcb (filesize, pos, eta, upcbClosure); - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; - size = GNUNET_ECRS_DBLOCK_SIZE; - if (size > filesize - pos) - { - size = filesize - pos; - memset (&db[1], 0, GNUNET_ECRS_DBLOCK_SIZE); - } - dblock->size = - htonl (sizeof (GNUNET_DatastoreValue) + size + - sizeof (GNUNET_EC_DBlock)); - if (size != READ (fd, &db[1], size)) - { - GNUNET_GE_LOG_STRERROR_FILE (ectx, - GNUNET_GE_ERROR | GNUNET_GE_USER | - GNUNET_GE_ADMIN | GNUNET_GE_BULK, - "READ", filename); - goto FAILURE; - } - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; - GNUNET_EC_file_block_get_key (db, size + sizeof (GNUNET_EC_DBlock), - &chk.key); - GNUNET_EC_file_block_get_query (db, size + sizeof (GNUNET_EC_DBlock), - &chk.query); - if (GNUNET_OK != pushBlock (sock, &chk, 0, /* dblocks are on level 0 */ - iblocks)) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - if (!wasIndexed) - { - if (GNUNET_OK == - GNUNET_EC_file_block_encode (db, size, &chk.query, &value)) - { - *value = *dblock; /* copy options! */ -#if STRICT_CHECKS - if (GNUNET_OK != GNUNET_FS_delete (sock, value)) - { - GNUNET_free (value); - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } -#else - GNUNET_FS_delete (sock, value); -#endif - GNUNET_free (value); - } - else - { - goto FAILURE; - } - } - pos += size; - now = GNUNET_get_time (); - eta = (GNUNET_CronTime) (start + - (((double) (now - start) / (double) pos)) - * (double) filesize); - } - if (tt != NULL) - if (GNUNET_OK != tt (ttClosure)) - goto FAILURE; - for (i = 0; i < treedepth; i++) - { - size = ntohl (iblocks[i]->size) - sizeof (GNUNET_DatastoreValue); - db = (GNUNET_EC_DBlock *) & iblocks[i][1]; - GNUNET_EC_file_block_get_key (db, size, &chk.key); - GNUNET_EC_file_block_get_query (db, size, &chk.query); - if (GNUNET_OK != pushBlock (sock, &chk, i + 1, iblocks)) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - GNUNET_EC_file_block_encode (db, size, &chk.query, &value); -#if STRICT_CHECKS - if (GNUNET_OK != GNUNET_FS_delete (sock, value)) - { - GNUNET_free (value); - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } -#else - GNUNET_FS_delete (sock, value); -#endif - GNUNET_free (value); - GNUNET_free (iblocks[i]); - iblocks[i] = NULL; - } - if (wasIndexed) - { - if (GNUNET_OK == undoSymlinking (ectx, filename, &fileId, sock)) - { - if (GNUNET_OK != - GNUNET_FS_unindex (sock, GNUNET_ECRS_DBLOCK_SIZE, &fileId)) - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - } - else - { - GNUNET_GE_BREAK (ectx, 0); - goto FAILURE; - } - } - GNUNET_free (iblocks[treedepth]); - /* free resources */ - GNUNET_free (iblocks); - GNUNET_free (dblock); - CLOSE (fd); - GNUNET_client_connection_destroy (sock); - return GNUNET_OK; -FAILURE: - for (i = 0; i <= treedepth; i++) - GNUNET_free_non_null (iblocks[i]); - GNUNET_free (iblocks); - GNUNET_free (dblock); - CLOSE (fd); - GNUNET_client_connection_destroy (sock); - return GNUNET_SYSERR; +/** + * If necessary, connect to the datastore and remove the KBlocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) +{ + const char *keyword; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + + if (NULL == uc->dsh) + uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); + if (NULL == uc->dsh) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if ( (NULL == uc->ksk_uri) || + (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) ) + { + unindex_finish (uc); + return; + } + /* FIXME: code duplication with fs_search.c here... */ + keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; + GNUNET_CRYPTO_hash (keyword, strlen (keyword), &uc->key); + pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&uc->key); + GNUNET_assert (pk != NULL); + GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); + GNUNET_CRYPTO_rsa_key_free (pk); + GNUNET_CRYPTO_hash (&pub, + sizeof (struct + GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &uc->query); + uc->first_uid = 0; + uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, + uc->roff++, + &uc->query, + GNUNET_BLOCK_TYPE_FS_KBLOCK, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &process_kblock_for_unindex, + uc); +} + + +/** + * Function called when the tree encoder has + * processed all blocks. Clean up. + * + * @param cls our unindexing context + * @param tc not used + */ +static void +unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_extract_keywords_ (uc); +} + + +/** + * Connect to the datastore and remove the blocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) +{ + if (NULL == uc->dsh) + uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); + if (NULL == uc->dsh) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->fh = + GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (NULL == uc->fh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to open file for unindexing.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->tc = + GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, + &unindex_process, &unindex_progress, + &unindex_extract_keywords); + GNUNET_FS_tree_encoder_next (uc->tc); +} + + +/** + * Function called once the hash of the file + * that is being unindexed has been computed. + * + * @param cls closure, unindex context + * @param file_id computed hash, NULL on error + */ +void +GNUNET_FS_unindex_process_hash_ (void *cls, const struct GNUNET_HashCode * file_id) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->fhc = NULL; + if (uc->state != UNINDEX_STATE_HASHING) + { + GNUNET_FS_unindex_stop (uc); + return; + } + if (file_id == NULL) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to compute hash of file.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->file_id = *file_id; + uc->state = UNINDEX_STATE_DS_REMOVE; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_remove_ (uc); +} + + +/** + * Create SUSPEND event for the given unindex operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for + */ +void +GNUNET_FS_unindex_signal_suspend_ (void *cls) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + /* FIXME: lots of duplication with unindex_stop here! */ + if (uc->dscan != NULL) + { + GNUNET_FS_directory_scan_abort (uc->dscan); + uc->dscan = NULL; + } + if (NULL != uc->dqe) + { + GNUNET_DATASTORE_cancel (uc->dqe); + uc->dqe = NULL; + } + if (uc->fhc != NULL) + { + GNUNET_CRYPTO_hash_file_cancel (uc->fhc); + uc->fhc = NULL; + } + if (NULL != uc->ksk_uri) + { + GNUNET_FS_uri_destroy (uc->ksk_uri); + uc->ksk_uri = NULL; + } + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client); + uc->client = NULL; + } + if (NULL != uc->dsh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + } + if (NULL != uc->tc) + { + GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); + uc->tc = NULL; + } + if (uc->fh != NULL) + { + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + } + GNUNET_FS_end_top (uc->h, uc->top); + pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + GNUNET_break (NULL == uc->client_info); + GNUNET_free (uc->filename); + GNUNET_free_non_null (uc->serialization); + GNUNET_free_non_null (uc->emsg); + GNUNET_free (uc); } -#endif + +/** + * Unindex a file. + * + * @param h handle to the file sharing subsystem + * @param filename file to unindex + * @param cctx initial value for the client context + * @return NULL on error, otherwise handle + */ +struct GNUNET_FS_UnindexContext * +GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, const char *filename, + void *cctx) +{ + struct GNUNET_FS_UnindexContext *ret; + struct GNUNET_FS_ProgressInfo pi; + uint64_t size; + + if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES, GNUNET_YES)) + return NULL; + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); + ret->h = h; + ret->filename = GNUNET_strdup (filename); + ret->start_time = GNUNET_TIME_absolute_get (); + ret->file_size = size; + ret->client_info = cctx; + GNUNET_FS_unindex_sync_ (ret); + pi.status = GNUNET_FS_STATUS_UNINDEX_START; + pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; + GNUNET_FS_unindex_make_status_ (&pi, ret, 0); + ret->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, filename, + HASHING_BLOCKSIZE, + &GNUNET_FS_unindex_process_hash_, ret); + ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, ret); + return ret; +} + + +/** + * Clean up after completion of an unindex operation. + * + * @param uc handle + */ +void +GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) +{ + struct GNUNET_FS_ProgressInfo pi; + + if (uc->dscan != NULL) + { + GNUNET_FS_directory_scan_abort (uc->dscan); + uc->dscan = NULL; + } + if (NULL != uc->dqe) + { + GNUNET_DATASTORE_cancel (uc->dqe); + uc->dqe = NULL; + } + if (uc->fhc != NULL) + { + GNUNET_CRYPTO_hash_file_cancel (uc->fhc); + uc->fhc = NULL; + } + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client); + uc->client = NULL; + } + if (NULL != uc->dsh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + } + if (NULL != uc->ksk_uri) + { + GNUNET_FS_uri_destroy (uc->ksk_uri); + uc->ksk_uri = NULL; + } + if (NULL != uc->tc) + { + GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); + uc->tc = NULL; + } + if (uc->fh != NULL) + { + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + } + GNUNET_FS_end_top (uc->h, uc->top); + if (uc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free (uc->serialization); + uc->serialization = NULL; + } + pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED; + pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + GNUNET_break (NULL == uc->client_info); + GNUNET_free_non_null (uc->emsg); + GNUNET_free (uc->filename); + GNUNET_free (uc); +} /* end of fs_unindex.c */