X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Ffs%2Ffs_unindex.c;h=71a1b9e0ba78c4195904693d9c3a113801e19091;hb=eef168bdbf40609b944dada415fc536e15cac494;hp=130b2eae2cf46913dc8debadcb2840abcf67b779;hpb=502af2167f7c218366666ca4944bd7cc54b5b19a;p=oweals%2Fgnunet.git diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index 130b2eae2..71a1b9e0b 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -28,10 +28,10 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" +#include "block_fs.h" -#define DEBUG_UNINDEX GNUNET_NO /** * Function called by the tree encoder to obtain @@ -68,7 +68,7 @@ unindex_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) /** - * Fill in all of the generic fields for + * Fill in all of the generic fields for * an unindex event and call the callback. * * @param pi structure to fill in @@ -84,8 +84,8 @@ GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, pi->value.unindex.cctx = uc->client_info; pi->value.unindex.filename = uc->filename; pi->value.unindex.size = uc->file_size; - pi->value.unindex.eta - = GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size); + pi->value.unindex.eta = + GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size); pi->value.unindex.duration = GNUNET_TIME_absolute_get_duration (uc->start_time); pi->value.unindex.completed = offset; @@ -105,9 +105,8 @@ GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, * @param depth depth of the block in the tree, 0 for DBLOCK */ static void -unindex_progress (void *cls, - uint64_t offset, - const void *pt_block, size_t pt_size, unsigned int depth) +unindex_progress (void *cls, uint64_t offset, const void *pt_block, + size_t pt_size, unsigned int depth) { struct GNUNET_FS_UnindexContext *uc = cls; struct GNUNET_FS_ProgressInfo pi; @@ -145,10 +144,11 @@ signal_unindex_error (struct GNUNET_FS_UnindexContext *uc) * * @param cls closure * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored * @param msg NULL on success, otherwise an error message */ static void -process_cont (void *cls, int success, const char *msg) +process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) { struct GNUNET_FS_UnindexContext *uc = cls; @@ -158,10 +158,8 @@ process_cont (void *cls, int success, const char *msg) signal_unindex_error (uc); return; } -#if DEBUG_UNINDEX GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Datastore REMOVE operation succeeded\n"); -#endif GNUNET_FS_tree_encoder_next (uc->tc); } @@ -181,11 +179,8 @@ process_cont (void *cls, int success, const char *msg) * @param block_size size of block (in bytes) */ static void -unindex_process (void *cls, - const struct ContentHashKey *chk, - uint64_t offset, - unsigned int depth, - enum GNUNET_BLOCK_Type type, +unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, + unsigned int depth, enum GNUNET_BLOCK_Type type, const void *block, uint16_t block_size) { struct GNUNET_FS_UnindexContext *uc = cls; @@ -205,16 +200,11 @@ unindex_process (void *cls, odb.file_id = uc->file_id; data = &odb; } -#if DEBUG_UNINDEX GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending REMOVE request to DATASTORE service\n"); -#endif - GNUNET_DATASTORE_remove (uc->dsh, - &chk->query, - size, - data, - -2, 1, + GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); + uc->chk = *chk; } @@ -233,7 +223,7 @@ process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) if (uc->client != NULL) { - GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + GNUNET_CLIENT_disconnect (uc->client); uc->client = NULL; } if (uc->state != UNINDEX_STATE_FS_NOTIFY) @@ -270,16 +260,15 @@ process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) /** - * Function called when the tree encoder has - * processed all blocks. Clean up. + * Function called when we are done with removing KBlocks. + * Disconnect from datastore and notify FS service about + * the unindex event. * - * @param cls our unindexing context - * @param tc not used + * @param uc our unindexing context */ static void -unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +unindex_finish (struct GNUNET_FS_UnindexContext *uc) { - struct GNUNET_FS_UnindexContext *uc = cls; char *emsg; struct GNUNET_FS_Uri *uri; struct UnindexMessage req; @@ -306,10 +295,8 @@ unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) signal_unindex_error (uc); return; } -#if DEBUG_UNINDEX GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending UNINDEX message to FS service\n"); -#endif req.header.size = htons (sizeof (struct UnindexMessage)); req.header.type = htons (GNUNET_MESSAGE_TYPE_FS_UNINDEX); req.reserved = 0; @@ -324,6 +311,297 @@ unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) } + +/** + * Function called by the directory scanner as we extract keywords + * that we will need to remove KBlocks. + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param filename which file we are making progress on + * @param is_directory GNUNET_YES if this is a directory, + * GNUNET_NO if this is a file + * GNUNET_SYSERR if it is neither (or unknown) + * @param reason kind of progress we are making + */ +static void +unindex_directory_scan_cb (void *cls, + const char *filename, + int is_directory, + enum GNUNET_FS_DirScannerProgressUpdateReason reason) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + static struct GNUNET_FS_ShareTreeItem * directory_scan_result; + + switch (reason) + { + case GNUNET_FS_DIRSCANNER_FINISHED: + directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan); + uc->dscan = NULL; + if (NULL != directory_scan_result->ksk_uri) + { + uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri); + uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS; + uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan.")); + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_remove_kblocks_ (uc); + } + else + { + unindex_finish (uc); + } + GNUNET_FS_share_tree_free (directory_scan_result); + break; + case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Internal error scanning `%s'.\n"), + uc->filename); + break; + default: + break; + } + +} + + +/** + * If necessary, connect to the datastore and remove the KBlocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc) +{ + char *ex; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex)) + ex = NULL; + uc->dscan = GNUNET_FS_directory_scan_start (uc->filename, + GNUNET_NO, ex, + &unindex_directory_scan_cb, + uc); + GNUNET_free_non_null (ex); +} + + +/** + * Continuation called to notify client about result of the remove + * operation for the KBlock. + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param success GNUNET_SYSERR on failure (including timeout/queue drop) + * GNUNET_NO if content was already there + * GNUNET_YES (or other positive value) on success + * @param min_expiration minimum expiration time required for 0-priority content to be stored + * by the datacache at this time, zero for unknown, forever if we have no + * space for 0-priority content + * @param msg NULL on success, otherwise an error message + */ +static void +continue_after_remove (void *cls, + int32_t success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->dqe = NULL; + if (success != GNUNET_YES) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to remove KBlock: %s\n"), + msg); + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); +} + + +/** + * Function called from datastore with result from us looking for + * a KBlock. There are four cases: + * 1) no result, means we move on to the next keyword + * 2) UID is the same as the first UID, means we move on to next keyword + * 3) KBlock for a different CHK, means we keep looking for more + * 4) KBlock is for our CHK, means we remove the block and then move + * on to the next keyword + * + * @param cls the 'struct GNUNET_FS_UnindexContext *' + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +process_kblock_for_unindex (void *cls, + const struct GNUNET_HashCode * key, + size_t size, const void *data, + enum GNUNET_BLOCK_Type type, + uint32_t priority, + uint32_t anonymity, + struct GNUNET_TIME_Absolute + expiration, uint64_t uid) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + const struct KBlock *kb; + struct GNUNET_FS_Uri *chk_uri; + + uc->dqe = NULL; + if (NULL == data) + { + /* no result */ + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); + return; + } + if (0 == uc->first_uid) + { + /* remember UID of first result to detect cycles */ + uc->first_uid = uid; + } + else if (uid == uc->first_uid) + { + /* no more additional results */ + uc->ksk_offset++; + GNUNET_FS_unindex_do_remove_kblocks_ (uc); + return; + } + GNUNET_assert (GNUNET_BLOCK_TYPE_FS_KBLOCK == type); + if (size < sizeof (struct KBlock)) + { + GNUNET_break (0); + goto get_next; + } + kb = data; + { + char pt[size - sizeof (struct KBlock)]; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + + GNUNET_CRYPTO_hash_to_aes_key (&uc->key, &skey, &iv); + if (-1 == + GNUNET_CRYPTO_aes_decrypt (&kb[1], size - sizeof (struct KBlock), &skey, + &iv, pt)) + { + GNUNET_break (0); + goto get_next; + } + if (NULL == memchr (pt, 0, sizeof (pt))) + { + GNUNET_break (0); + goto get_next; + } + chk_uri = GNUNET_FS_uri_parse (pt, NULL); + if (NULL == chk_uri) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to parse URI `%s' from KBlock!\n"), + pt); + GNUNET_break (0); + goto get_next; + } + } + if (0 != memcmp (&uc->chk, + &chk_uri->data.chk.chk, + sizeof (struct ContentHashKey))) + { + /* different CHK, ignore */ + GNUNET_FS_uri_destroy (chk_uri); + goto get_next; + } + GNUNET_FS_uri_destroy (chk_uri); + /* matches! */ + uc->dqe = GNUNET_DATASTORE_remove (uc->dsh, + key, size, data, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &continue_after_remove, + uc); + return; + get_next: + uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, + uc->roff++, + &uc->query, + GNUNET_BLOCK_TYPE_FS_KBLOCK, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &process_kblock_for_unindex, + uc); +} + + +/** + * If necessary, connect to the datastore and remove the KBlocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc) +{ + const char *keyword; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + + if (NULL == uc->dsh) + uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); + if (NULL == uc->dsh) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if ( (NULL == uc->ksk_uri) || + (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) ) + { + unindex_finish (uc); + return; + } + /* FIXME: code duplication with fs_search.c here... */ + keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1]; + GNUNET_CRYPTO_hash (keyword, strlen (keyword), &uc->key); + pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&uc->key); + GNUNET_assert (pk != NULL); + GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); + GNUNET_CRYPTO_rsa_key_free (pk); + GNUNET_CRYPTO_hash (&pub, + sizeof (struct + GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &uc->query); + uc->first_uid = 0; + uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh, + uc->roff++, + &uc->query, + GNUNET_BLOCK_TYPE_FS_KBLOCK, + 0 /* priority */, 1 /* queue size */, + GNUNET_TIME_UNIT_FOREVER_REL, + &process_kblock_for_unindex, + uc); +} + + +/** + * Function called when the tree encoder has + * processed all blocks. Clean up. + * + * @param cls our unindexing context + * @param tc not used + */ +static void +unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_extract_keywords_ (uc); +} + + /** * Connect to the datastore and remove the blocks. * @@ -332,7 +610,8 @@ unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) void GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) { - uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); + if (NULL == uc->dsh) + uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); if (NULL == uc->dsh) { uc->state = UNINDEX_STATE_ERROR; @@ -341,8 +620,9 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) signal_unindex_error (uc); return; } - uc->fh = GNUNET_DISK_file_open (uc->filename, - GNUNET_DISK_OPEN_READ, GNUNET_DISK_PERM_NONE); + uc->fh = + GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); if (NULL == uc->fh) { GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); @@ -353,12 +633,10 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) signal_unindex_error (uc); return; } - uc->tc = GNUNET_FS_tree_encoder_create (uc->h, - uc->file_size, - uc, - &unindex_reader, - &unindex_process, - &unindex_progress, &unindex_finish); + uc->tc = + GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, + &unindex_process, &unindex_progress, + &unindex_extract_keywords); GNUNET_FS_tree_encoder_next (uc->tc); } @@ -371,7 +649,7 @@ GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) * @param file_id computed hash, NULL on error */ void -GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id) +GNUNET_FS_unindex_process_hash_ (void *cls, const struct GNUNET_HashCode * file_id) { struct GNUNET_FS_UnindexContext *uc = cls; @@ -408,14 +686,30 @@ GNUNET_FS_unindex_signal_suspend_ (void *cls) struct GNUNET_FS_UnindexContext *uc = cls; struct GNUNET_FS_ProgressInfo pi; + /* FIXME: lots of duplication with unindex_stop here! */ + if (uc->dscan != NULL) + { + GNUNET_FS_directory_scan_abort (uc->dscan); + uc->dscan = NULL; + } + if (NULL != uc->dqe) + { + GNUNET_DATASTORE_cancel (uc->dqe); + uc->dqe = NULL; + } if (uc->fhc != NULL) { GNUNET_CRYPTO_hash_file_cancel (uc->fhc); uc->fhc = NULL; } + if (NULL != uc->ksk_uri) + { + GNUNET_FS_uri_destroy (uc->ksk_uri); + uc->ksk_uri = NULL; + } if (uc->client != NULL) { - GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + GNUNET_CLIENT_disconnect (uc->client); uc->client = NULL; } if (NULL != uc->dsh) @@ -436,8 +730,8 @@ GNUNET_FS_unindex_signal_suspend_ (void *cls) GNUNET_FS_end_top (uc->h, uc->top); pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND; GNUNET_FS_unindex_make_status_ (&pi, uc, - (uc->state == UNINDEX_STATE_COMPLETE) - ? uc->file_size : 0); + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); GNUNET_break (NULL == uc->client_info); GNUNET_free (uc->filename); GNUNET_free_non_null (uc->serialization); @@ -452,17 +746,17 @@ GNUNET_FS_unindex_signal_suspend_ (void *cls) * @param h handle to the file sharing subsystem * @param filename file to unindex * @param cctx initial value for the client context - * @return NULL on error, otherwise handle + * @return NULL on error, otherwise handle */ struct GNUNET_FS_UnindexContext * -GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, - const char *filename, void *cctx) +GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, const char *filename, + void *cctx) { struct GNUNET_FS_UnindexContext *ret; struct GNUNET_FS_ProgressInfo pi; uint64_t size; - if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES)) + if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES, GNUNET_YES)) return NULL; ret = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); ret->h = h; @@ -474,10 +768,10 @@ GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, pi.status = GNUNET_FS_STATUS_UNINDEX_START; pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; GNUNET_FS_unindex_make_status_ (&pi, ret, 0); - ret->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, - filename, - HASHING_BLOCKSIZE, - &GNUNET_FS_unindex_process_hash_, ret); + ret->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, filename, + HASHING_BLOCKSIZE, + &GNUNET_FS_unindex_process_hash_, ret); ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, ret); return ret; } @@ -493,6 +787,16 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) { struct GNUNET_FS_ProgressInfo pi; + if (uc->dscan != NULL) + { + GNUNET_FS_directory_scan_abort (uc->dscan); + uc->dscan = NULL; + } + if (NULL != uc->dqe) + { + GNUNET_DATASTORE_cancel (uc->dqe); + uc->dqe = NULL; + } if (uc->fhc != NULL) { GNUNET_CRYPTO_hash_file_cancel (uc->fhc); @@ -500,7 +804,7 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) } if (uc->client != NULL) { - GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + GNUNET_CLIENT_disconnect (uc->client); uc->client = NULL; } if (NULL != uc->dsh) @@ -508,6 +812,11 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); uc->dsh = NULL; } + if (NULL != uc->ksk_uri) + { + GNUNET_FS_uri_destroy (uc->ksk_uri); + uc->ksk_uri = NULL; + } if (NULL != uc->tc) { GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); @@ -529,9 +838,10 @@ GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED; pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; GNUNET_FS_unindex_make_status_ (&pi, uc, - (uc->state == UNINDEX_STATE_COMPLETE) - ? uc->file_size : 0); + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); GNUNET_break (NULL == uc->client_info); + GNUNET_free_non_null (uc->emsg); GNUNET_free (uc->filename); GNUNET_free (uc); }