From: Christian Grothoff Date: Tue, 26 Jan 2010 09:53:40 +0000 (+0000) Subject: moving indexing out of main file X-Git-Tag: initial-import-from-subversion-38251~22850 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=68907af3eb39fb03936a226f340d469af10d5014;p=oweals%2Fgnunet.git moving indexing out of main file --- diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 95212fe5e..70897c5ef 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -84,7 +84,8 @@ gnunet_search_LDADD = \ $(GN_LIBINTL) gnunet_service_fs_SOURCES = \ - gnunet-service-fs.c + gnunet-service-fs.c \ + gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h gnunet_service_fs_LDADD = \ $(top_builddir)/src/fs/libgnunetfs.la \ $(top_builddir)/src/datastore/libgnunetdatastore.la \ diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index fa83eb889..0339567f3 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -45,42 +45,12 @@ #include "gnunet_protocols.h" #include "gnunet_signatures.h" #include "gnunet_util_lib.h" +#include "gnunet-service-fs_indexing.h" #include "fs.h" #define DEBUG_FS GNUNET_NO -/** - * In-memory information about indexed files (also available - * on-disk). - */ -struct IndexInfo -{ - - /** - * This is a linked list. - */ - struct IndexInfo *next; - - /** - * Name of the indexed file. Memory allocated - * at the end of this struct (do not free). - */ - const char *filename; - - /** - * Context for transmitting confirmation to client, - * NULL if we've done this already. - */ - struct GNUNET_SERVER_TransmitContext *tc; - - /** - * Hash of the contents of the file. - */ - GNUNET_HashCode file_id; - -}; - /** * Signature of a function that is called whenever a datastore @@ -359,20 +329,26 @@ struct ProcessGetContext /** - * Information we keep for each pending reply. + * Information we keep for each pending reply. The + * actual message follows at the end of this struct. */ -struct PendingReply +struct PendingMessage { /** * This is a linked list. */ - struct PendingReply *next; + struct PendingMessage *next; /** * Size of the reply; actual reply message follows * at the end of this struct. */ size_t msize; + + /** + * How important is this message for us? + */ + uint32_t priority; }; @@ -426,7 +402,7 @@ struct PendingRequest * (typically non-null only if we have a pending transmission * request with the client or the respective peer). */ - struct PendingReply *replies_pending; + struct PendingMessage *replies_pending; /** * Pending transmission request with the core service for the target @@ -678,6 +654,18 @@ struct ConnectedPeer */ struct GNUNET_TIME_Relative avg_delay; + /** + * Handle for an active request for transmission to this + * peer, or NULL. + */ + struct GNUNET_CORE_PeerRequestHandle *prh; + + /** + * Messages (replies, queries, content migration) we would like to + * send to this peer in the near future. Sorted by priority. + */ + struct PendingMessage *pending_messages; + /** * Average priority of successful replies. Calculated * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n @@ -690,10 +678,9 @@ struct ConnectedPeer GNUNET_PEER_Id pid; /** - * Number of requests we have currently pending - * with this peer (that is, requests that were - * transmitted so recently that we would not retransmit - * them right now). + * Number of requests we have currently pending with this peer (that + * is, requests that were transmitted so recently that we would not + * retransmit them right now). */ unsigned int pending_requests; @@ -752,18 +739,6 @@ static struct DatastoreRequestQueue *drq_head; */ static struct DatastoreRequestQueue *drq_tail; -/** - * Linked list of indexed files. - */ -static struct IndexInfo *indexed_files; - -/** - * Maps hash over content of indexed files to the respective filename. - * The filenames are pointers into the indexed_files linked list and - * do not need to be freed. - */ -static struct GNUNET_CONTAINER_MultiHashMap *ifm; - /** * Map of query hash codes to requests. */ @@ -803,403 +778,8 @@ static struct GNUNET_CONTAINER_MultiHashMap *connected_peers; static uint64_t max_pending_requests = 32; -/** - * Write the current index information list to disk. - */ -static void -write_index_list () -{ - struct GNUNET_BIO_WriteHandle *wh; - char *fn; - struct IndexInfo *pos; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (cfg, - "FS", - "INDEXDB", - &fn)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _("Configuration option `%s' in section `%s' missing.\n"), - "INDEXDB", - "FS"); - return; - } - wh = GNUNET_BIO_write_open (fn); - if (NULL == wh) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _("Could not open `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - pos = indexed_files; - while (pos != NULL) - { - if ( (GNUNET_OK != - GNUNET_BIO_write (wh, - &pos->file_id, - sizeof (GNUNET_HashCode))) || - (GNUNET_OK != - GNUNET_BIO_write_string (wh, - pos->filename)) ) - break; - pos = pos->next; - } - if (GNUNET_OK != - GNUNET_BIO_write_close (wh)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _("Error writing `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - GNUNET_free (fn); -} - - -/** - * Read index information from disk. - */ -static void -read_index_list () -{ - struct GNUNET_BIO_ReadHandle *rh; - char *fn; - struct IndexInfo *pos; - char *fname; - GNUNET_HashCode hc; - size_t slen; - char *emsg; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (cfg, - "FS", - "INDEXDB", - &fn)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _("Configuration option `%s' in section `%s' missing.\n"), - "INDEXDB", - "FS"); - return; - } - if (GNUNET_NO == GNUNET_DISK_file_test (fn)) - { - /* no index info yet */ - GNUNET_free (fn); - return; - } - rh = GNUNET_BIO_read_open (fn); - if (NULL == rh) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - _("Could not open `%s'.\n"), - fn); - GNUNET_free (fn); - return; - } - - while ( (GNUNET_OK == - GNUNET_BIO_read (rh, - "Hash of indexed file", - &hc, - sizeof (GNUNET_HashCode))) && - (GNUNET_OK == - GNUNET_BIO_read_string (rh, - "Name of indexed file", - &fname, - 1024 * 16)) ) - { - slen = strlen (fname) + 1; - pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen); - pos->file_id = hc; - pos->filename = (const char *) &pos[1]; - memcpy (&pos[1], fname, slen); - if (GNUNET_SYSERR == - GNUNET_CONTAINER_multihashmap_put (ifm, - &hc, - (void*) pos->filename, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) - { - GNUNET_free (pos); - } - else - { - pos->next = indexed_files; - indexed_files = pos; - } - GNUNET_free (fname); - } - if (GNUNET_OK != - GNUNET_BIO_read_close (rh, &emsg)) - GNUNET_free (emsg); - GNUNET_free (fn); -} - - -/** - * We've validated the hash of the file we're about to - * index. Signal success to the client and update - * our internal data structures. - * - * @param ii the index info entry for the request - */ -static void -signal_index_ok (struct IndexInfo *ii) -{ - if (GNUNET_SYSERR == - GNUNET_CONTAINER_multihashmap_put (ifm, - &ii->file_id, - (void*) ii->filename, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"), - ii->filename, - (const char*) GNUNET_CONTAINER_multihashmap_get (ifm, - &ii->file_id)); - GNUNET_SERVER_transmit_context_append_data (ii->tc, - NULL, 0, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); - GNUNET_SERVER_transmit_context_run (ii->tc, - GNUNET_TIME_UNIT_MINUTES); - GNUNET_free (ii); - return; - } - ii->next = indexed_files; - indexed_files = ii; - write_index_list (); - GNUNET_SERVER_transmit_context_append_data (ii->tc, - NULL, 0, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); - GNUNET_SERVER_transmit_context_run (ii->tc, - GNUNET_TIME_UNIT_MINUTES); - ii->tc = NULL; -} - - -/** - * Function called once the hash computation over an - * indexed file has completed. - * - * @param cls closure, our publishing context - * @param res resulting hash, NULL on error - */ -static void -hash_for_index_val (void *cls, - const GNUNET_HashCode * - res) -{ - struct IndexInfo *ii = cls; - - if ( (res == NULL) || - (0 != memcmp (res, - &ii->file_id, - sizeof(GNUNET_HashCode))) ) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Hash mismatch trying to index file `%s' which has hash `%s'\n"), - ii->filename, - GNUNET_h2s (res)); -#if DEBUG_FS - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Wanted `%s'\n", - GNUNET_h2s (&ii->file_id)); -#endif - GNUNET_SERVER_transmit_context_append_data (ii->tc, - NULL, 0, - GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED); - GNUNET_SERVER_transmit_context_run (ii->tc, - GNUNET_TIME_UNIT_MINUTES); - GNUNET_free (ii); - return; - } - signal_index_ok (ii); -} - - -/** - * Handle INDEX_START-message. - * - * @param cls closure - * @param client identification of the client - * @param message the actual message - */ -static void -handle_index_start (void *cls, - struct GNUNET_SERVER_Client *client, - const struct GNUNET_MessageHeader *message) -{ - const struct IndexStartMessage *ism; - const char *fn; - uint16_t msize; - struct IndexInfo *ii; - size_t slen; - uint32_t dev; - uint64_t ino; - uint32_t mydev; - uint64_t myino; - - msize = ntohs(message->size); - if ( (msize <= sizeof (struct IndexStartMessage)) || - ( ((const char *)message)[msize-1] != '\0') ) - { - GNUNET_break (0); - GNUNET_SERVER_receive_done (client, - GNUNET_SYSERR); - return; - } - ism = (const struct IndexStartMessage*) message; - fn = (const char*) &ism[1]; - dev = ntohl (ism->device); - ino = GNUNET_ntohll (ism->inode); - ism = (const struct IndexStartMessage*) message; - slen = strlen (fn) + 1; - ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen); - ii->filename = (const char*) &ii[1]; - memcpy (&ii[1], fn, slen); - ii->file_id = ism->file_id; - ii->tc = GNUNET_SERVER_transmit_context_create (client); - if ( ( (dev != 0) || - (ino != 0) ) && - (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn, - &mydev, - &myino)) && - ( (dev == mydev) && - (ino == myino) ) ) - { - /* fast validation OK! */ - signal_index_ok (ii); - return; - } -#if DEBUG_FS - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n", - (unsigned long long) ino, - (unsigned long long) myino, - (unsigned int) dev, - (unsigned int) mydev); -#endif - /* slow validation, need to hash full file (again) */ - GNUNET_CRYPTO_hash_file (sched, - GNUNET_SCHEDULER_PRIORITY_IDLE, - fn, - HASHING_BLOCKSIZE, - &hash_for_index_val, - ii); -} - - -/** - * Handle INDEX_LIST_GET-message. - * - * @param cls closure - * @param client identification of the client - * @param message the actual message - */ -static void -handle_index_list_get (void *cls, - struct GNUNET_SERVER_Client *client, - const struct GNUNET_MessageHeader *message) -{ - struct GNUNET_SERVER_TransmitContext *tc; - struct IndexInfoMessage *iim; - char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE]; - size_t slen; - const char *fn; - struct IndexInfo *pos; - - tc = GNUNET_SERVER_transmit_context_create (client); - iim = (struct IndexInfoMessage*) buf; - pos = indexed_files; - while (NULL != pos) - { - fn = pos->filename; - slen = strlen (fn) + 1; - if (slen + sizeof (struct IndexInfoMessage) > - GNUNET_SERVER_MAX_MESSAGE_SIZE) - { - GNUNET_break (0); - break; - } - iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY); - iim->header.size = htons (slen + sizeof (struct IndexInfoMessage)); - iim->reserved = 0; - iim->file_id = pos->file_id; - memcpy (&iim[1], fn, slen); - GNUNET_SERVER_transmit_context_append_message (tc, - &iim->header); - pos = pos->next; - } - GNUNET_SERVER_transmit_context_append_data (tc, - NULL, 0, - GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END); - GNUNET_SERVER_transmit_context_run (tc, - GNUNET_TIME_UNIT_MINUTES); -} -/** - * Handle UNINDEX-message. - * - * @param cls closure - * @param client identification of the client - * @param message the actual message - */ -static void -handle_unindex (void *cls, - struct GNUNET_SERVER_Client *client, - const struct GNUNET_MessageHeader *message) -{ - const struct UnindexMessage *um; - struct IndexInfo *pos; - struct IndexInfo *prev; - struct IndexInfo *next; - struct GNUNET_SERVER_TransmitContext *tc; - int found; - - um = (const struct UnindexMessage*) message; - found = GNUNET_NO; - prev = NULL; - pos = indexed_files; - while (NULL != pos) - { - next = pos->next; - if (0 == memcmp (&pos->file_id, - &um->file_id, - sizeof (GNUNET_HashCode))) - { - if (prev == NULL) - indexed_files = next; - else - prev->next = next; - GNUNET_free (pos); - found = GNUNET_YES; - } - else - { - prev = pos; - } - pos = next; - } -#if DEBUG_FS - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Client requested unindexing of file `%s': %s\n", - GNUNET_h2s (&um->file_id), - found ? "found" : "not found"); -#endif - if (GNUNET_YES == found) - write_index_list (); - tc = GNUNET_SERVER_transmit_context_create (client); - GNUNET_SERVER_transmit_context_append_data (tc, - NULL, 0, - GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK); - GNUNET_SERVER_transmit_context_run (tc, - GNUNET_TIME_UNIT_MINUTES); -} /** @@ -1369,27 +949,6 @@ transmit_local_result (void *cls, } -/** - * Continuation called from datastore's remove - * function. - * - * @param cls unused - * @param success did the deletion work? - * @param msg error message - */ -static void -remove_cont (void *cls, - int success, - const char *msg) -{ - if (GNUNET_OK != success) - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Failed to delete bogus block: %s\n"), - msg); - GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); -} - - /** * Mingle hash with the mingle_number to * produce different bits. @@ -1408,130 +967,6 @@ mingle_hash (const GNUNET_HashCode * in, } -/** - * We've received an on-demand encoded block - * from the datastore. Attempt to do on-demand - * encoding and (if successful), call the - * continuation with the resulting block. On - * error, clean up and ask the datastore for - * more results. - * - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - * @param cont function to call with the actual block - * @param cont_cls closure for cont - */ -static void -handle_on_demand_block (const GNUNET_HashCode * key, - uint32_t size, - const void *data, - uint32_t type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, uint64_t uid, - GNUNET_DATASTORE_Iterator cont, - void *cont_cls) -{ - const struct OnDemandBlock *odb; - GNUNET_HashCode nkey; - struct GNUNET_CRYPTO_AesSessionKey skey; - struct GNUNET_CRYPTO_AesInitializationVector iv; - GNUNET_HashCode query; - ssize_t nsize; - char ndata[DBLOCK_SIZE]; - char edata[DBLOCK_SIZE]; - const char *fn; - struct GNUNET_DISK_FileHandle *fh; - uint64_t off; - - if (size != sizeof (struct OnDemandBlock)) - { - GNUNET_break (0); - GNUNET_DATASTORE_remove (dsh, - key, - size, - data, - &remove_cont, - NULL, - GNUNET_TIME_UNIT_FOREVER_REL); - return; - } - odb = (const struct OnDemandBlock*) data; - off = GNUNET_ntohll (odb->offset); - fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm, - &odb->file_id); - fh = NULL; - if ( (NULL == fn) || - (NULL == (fh = GNUNET_DISK_file_open (fn, - GNUNET_DISK_OPEN_READ, - GNUNET_DISK_PERM_NONE))) || - (off != - GNUNET_DISK_file_seek (fh, - off, - GNUNET_DISK_SEEK_SET)) || - (-1 == - (nsize = GNUNET_DISK_file_read (fh, - ndata, - sizeof (ndata)))) ) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Could not access indexed file `%s' at offset %llu: %s\n"), - GNUNET_h2s (&odb->file_id), - (unsigned long long) off, - STRERROR (errno)); - if (fh != NULL) - GNUNET_DISK_file_close (fh); - /* FIXME: if this happens often, we need - to remove the OnDemand block from the DS! */ - GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); - return; - } - GNUNET_DISK_file_close (fh); - GNUNET_CRYPTO_hash (ndata, - nsize, - &nkey); - GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv); - GNUNET_CRYPTO_aes_encrypt (ndata, - nsize, - &skey, - &iv, - edata); - GNUNET_CRYPTO_hash (edata, - nsize, - &query); - if (0 != memcmp (&query, - key, - sizeof (GNUNET_HashCode))) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Indexed file `%s' changed at offset %llu\n"), - fn, - (unsigned long long) off); - /* FIXME: if this happens often, we need - to remove the OnDemand block from the DS! */ - GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); - return; - } - cont (cont_cls, - key, - nsize, - edata, - GNUNET_DATASTORE_BLOCKTYPE_DBLOCK, - priority, - anonymity, - expiration, - uid); -} - - /** * How many bytes should a bloomfilter be if we have already seen * entry_count responses? Note that BLOOMFILTER_K gives us the number @@ -2001,10 +1436,11 @@ process_local_get_result (void *cls, "Received on-demand block for `%s' from local datastore, fetching data.\n", GNUNET_h2s (&lgc->query)); #endif - handle_on_demand_block (key, size, data, type, priority, - anonymity, expiration, uid, - &process_local_get_result, - lgc); + GNUNET_FS_handle_on_demand_block (key, size, data, type, priority, + anonymity, expiration, uid, + dsh, + &process_local_get_result, + lgc); return; } if ( (type != lgc->type) && @@ -2184,11 +1620,11 @@ handle_start_search (void *cls, * service. */ static struct GNUNET_SERVER_MessageHandler handlers[] = { - {&handle_index_start, NULL, + {&GNUNET_FS_handle_index_start, NULL, GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0}, - {&handle_index_list_get, NULL, + {&GNUNET_FS_handle_index_list_get, NULL, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, sizeof(struct GNUNET_MessageHeader) }, - {&handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX, + {&GNUNET_FS_handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX, sizeof (struct UnindexMessage) }, {&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH, 0 }, @@ -2205,7 +1641,7 @@ static struct GNUNET_SERVER_MessageHandler handlers[] = { static void destroy_pending_request (struct PendingRequest *pr) { - struct PendingReply *reply; + struct PendingMessage *reply; struct ClientList *cl; GNUNET_CONTAINER_multihashmap_remove (requests_by_query, @@ -2333,8 +1769,6 @@ static void shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - struct IndexInfo *pos; - if (NULL != core) { GNUNET_CORE_disconnect (core); @@ -2362,13 +1796,6 @@ shutdown_task (void *cls, // (or do we get disconnect notifications?) GNUNET_CONTAINER_multihashmap_destroy (connected_peers); connected_peers = NULL; - GNUNET_CONTAINER_multihashmap_destroy (ifm); - ifm = NULL; - while (NULL != (pos = indexed_files)) - { - indexed_files = pos->next; - GNUNET_free (pos); - } } @@ -2623,10 +2050,10 @@ process_p2p_get_result (void *cls, } if (type == GNUNET_DATASTORE_BLOCKTYPE_ONDEMAND) { - handle_on_demand_block (key, size, data, type, priority, - anonymity, expiration, uid, - &process_p2p_get_result, - pgc); + GNUNET_FS_handle_on_demand_block (key, size, data, type, priority, + anonymity, expiration, uid, dsh, + &process_p2p_get_result, + pgc); return; } /* check for duplicates */ @@ -2961,7 +2388,7 @@ transmit_result (void *cls, { struct PendingRequest *pr = cls; char *cbuf = buf; - struct PendingReply *reply; + struct PendingMessage *reply; size_t ret; ret = 0; @@ -2995,7 +2422,7 @@ process_reply (void *cls, struct ProcessReplyClosure *prq = cls; struct PendingRequest *pr = value; struct PendingRequest *eer; - struct PendingReply *reply; + struct PendingMessage *reply; struct PutMessage *pm; struct ContentMessage *cm; GNUNET_HashCode chash; @@ -3046,7 +2473,7 @@ process_reply (void *cls, if (pr->client == NULL) { msize = sizeof (struct ContentMessage) + prq->size; - reply = GNUNET_malloc (msize + sizeof (struct PendingReply)); + reply = GNUNET_malloc (msize + sizeof (struct PendingMessage)); reply->msize = msize; cm = (struct ContentMessage*) &reply[1]; cm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CONTENT); @@ -3084,7 +2511,7 @@ process_reply (void *cls, else { msize = sizeof (struct PutMessage) + prq->size; - reply = GNUNET_malloc (msize + sizeof (struct PendingReply)); + reply = GNUNET_malloc (msize + sizeof (struct PendingMessage)); reply->msize = msize; reply->next = pr->replies_pending; pm = (struct PutMessage*) &reply[1]; @@ -3319,12 +2746,12 @@ run (void *cls, sched = s; cfg = c; - ifm = GNUNET_CONTAINER_multihashmap_create (128); + requests_by_query = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config requests_by_peer = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config connected_peers = GNUNET_CONTAINER_multihashmap_create (64); requests_by_expiration = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); - read_index_list (); + GNUNET_FS_init_indexing (sched, cfg); dsh = GNUNET_DATASTORE_connect (cfg, sched); core = GNUNET_CORE_connect (sched, diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c new file mode 100644 index 000000000..ebd7114d3 --- /dev/null +++ b/src/fs/gnunet-service-fs_indexing.c @@ -0,0 +1,688 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_indexing.c + * @brief program that provides indexing functions of the file-sharing service + * @author Christian Grothoff + * + * TODO: + * - indexed files/blocks not removed on errors + */ +#include "platform.h" +#include +#include "gnunet_core_service.h" +#include "gnunet_datastore_service.h" +#include "gnunet_peer_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet-service-fs_indexing.h" +#include "fs.h" + +#define DEBUG_FS GNUNET_NO + + +/** + * In-memory information about indexed files (also available + * on-disk). + */ +struct IndexInfo +{ + + /** + * This is a linked list. + */ + struct IndexInfo *next; + + /** + * Name of the indexed file. Memory allocated + * at the end of this struct (do not free). + */ + const char *filename; + + /** + * Context for transmitting confirmation to client, + * NULL if we've done this already. + */ + struct GNUNET_SERVER_TransmitContext *tc; + + /** + * Hash of the contents of the file. + */ + GNUNET_HashCode file_id; + +}; + + +/** + * Linked list of indexed files. + */ +static struct IndexInfo *indexed_files; + +/** + * Maps hash over content of indexed files to the respective filename. + * The filenames are pointers into the indexed_files linked list and + * do not need to be freed. + */ +static struct GNUNET_CONTAINER_MultiHashMap *ifm; + +/** + * Our scheduler. + */ +static struct GNUNET_SCHEDULER_Handle *sched; + +/** + * Our configuration. + */ +const struct GNUNET_CONFIGURATION_Handle *cfg; + + +/** + * Write the current index information list to disk. + */ +static void +write_index_list () +{ + struct GNUNET_BIO_WriteHandle *wh; + char *fn; + struct IndexInfo *pos; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, + "FS", + "INDEXDB", + &fn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Configuration option `%s' in section `%s' missing.\n"), + "INDEXDB", + "FS"); + return; + } + wh = GNUNET_BIO_write_open (fn); + if (NULL == wh) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Could not open `%s'.\n"), + fn); + GNUNET_free (fn); + return; + } + pos = indexed_files; + while (pos != NULL) + { + if ( (GNUNET_OK != + GNUNET_BIO_write (wh, + &pos->file_id, + sizeof (GNUNET_HashCode))) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + pos->filename)) ) + break; + pos = pos->next; + } + if (GNUNET_OK != + GNUNET_BIO_write_close (wh)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Error writing `%s'.\n"), + fn); + GNUNET_free (fn); + return; + } + GNUNET_free (fn); +} + + +/** + * Read index information from disk. + */ +static void +read_index_list () +{ + struct GNUNET_BIO_ReadHandle *rh; + char *fn; + struct IndexInfo *pos; + char *fname; + GNUNET_HashCode hc; + size_t slen; + char *emsg; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, + "FS", + "INDEXDB", + &fn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Configuration option `%s' in section `%s' missing.\n"), + "INDEXDB", + "FS"); + return; + } + if (GNUNET_NO == GNUNET_DISK_file_test (fn)) + { + /* no index info yet */ + GNUNET_free (fn); + return; + } + rh = GNUNET_BIO_read_open (fn); + if (NULL == rh) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Could not open `%s'.\n"), + fn); + GNUNET_free (fn); + return; + } + + while ( (GNUNET_OK == + GNUNET_BIO_read (rh, + "Hash of indexed file", + &hc, + sizeof (GNUNET_HashCode))) && + (GNUNET_OK == + GNUNET_BIO_read_string (rh, + "Name of indexed file", + &fname, + 1024 * 16)) ) + { + slen = strlen (fname) + 1; + pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen); + pos->file_id = hc; + pos->filename = (const char *) &pos[1]; + memcpy (&pos[1], fname, slen); + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_put (ifm, + &hc, + (void*) pos->filename, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + GNUNET_free (pos); + } + else + { + pos->next = indexed_files; + indexed_files = pos; + } + GNUNET_free (fname); + } + if (GNUNET_OK != + GNUNET_BIO_read_close (rh, &emsg)) + GNUNET_free (emsg); + GNUNET_free (fn); +} + + +/** + * We've validated the hash of the file we're about to index. Signal + * success to the client and update our internal data structures. + * + * @param ii the index info entry for the request + */ +static void +signal_index_ok (struct IndexInfo *ii) +{ + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_put (ifm, + &ii->file_id, + (void*) ii->filename, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"), + ii->filename, + (const char*) GNUNET_CONTAINER_multihashmap_get (ifm, + &ii->file_id)); + GNUNET_SERVER_transmit_context_append_data (ii->tc, + NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); + GNUNET_SERVER_transmit_context_run (ii->tc, + GNUNET_TIME_UNIT_MINUTES); + GNUNET_free (ii); + return; + } + ii->next = indexed_files; + indexed_files = ii; + write_index_list (); + GNUNET_SERVER_transmit_context_append_data (ii->tc, + NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); + GNUNET_SERVER_transmit_context_run (ii->tc, + GNUNET_TIME_UNIT_MINUTES); + ii->tc = NULL; +} + + +/** + * Function called once the hash computation over an + * indexed file has completed. + * + * @param cls closure, our publishing context + * @param res resulting hash, NULL on error + */ +static void +hash_for_index_val (void *cls, + const GNUNET_HashCode * + res) +{ + struct IndexInfo *ii = cls; + + if ( (res == NULL) || + (0 != memcmp (res, + &ii->file_id, + sizeof(GNUNET_HashCode))) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Hash mismatch trying to index file `%s' which has hash `%s'\n"), + ii->filename, + GNUNET_h2s (res)); +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Wanted `%s'\n", + GNUNET_h2s (&ii->file_id)); +#endif + GNUNET_SERVER_transmit_context_append_data (ii->tc, + NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED); + GNUNET_SERVER_transmit_context_run (ii->tc, + GNUNET_TIME_UNIT_MINUTES); + GNUNET_free (ii); + return; + } + signal_index_ok (ii); +} + + +/** + * Handle INDEX_START-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_start (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct IndexStartMessage *ism; + const char *fn; + uint16_t msize; + struct IndexInfo *ii; + size_t slen; + uint32_t dev; + uint64_t ino; + uint32_t mydev; + uint64_t myino; + + msize = ntohs(message->size); + if ( (msize <= sizeof (struct IndexStartMessage)) || + ( ((const char *)message)[msize-1] != '\0') ) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, + GNUNET_SYSERR); + return; + } + ism = (const struct IndexStartMessage*) message; + fn = (const char*) &ism[1]; + dev = ntohl (ism->device); + ino = GNUNET_ntohll (ism->inode); + ism = (const struct IndexStartMessage*) message; + slen = strlen (fn) + 1; + ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen); + ii->filename = (const char*) &ii[1]; + memcpy (&ii[1], fn, slen); + ii->file_id = ism->file_id; + ii->tc = GNUNET_SERVER_transmit_context_create (client); + if ( ( (dev != 0) || + (ino != 0) ) && + (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn, + &mydev, + &myino)) && + ( (dev == mydev) && + (ino == myino) ) ) + { + /* fast validation OK! */ + signal_index_ok (ii); + return; + } +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n", + (unsigned long long) ino, + (unsigned long long) myino, + (unsigned int) dev, + (unsigned int) mydev); +#endif + /* slow validation, need to hash full file (again) */ + GNUNET_CRYPTO_hash_file (sched, + GNUNET_SCHEDULER_PRIORITY_IDLE, + fn, + HASHING_BLOCKSIZE, + &hash_for_index_val, + ii); +} + + +/** + * Handle INDEX_LIST_GET-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_list_get (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + struct GNUNET_SERVER_TransmitContext *tc; + struct IndexInfoMessage *iim; + char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE]; + size_t slen; + const char *fn; + struct IndexInfo *pos; + + tc = GNUNET_SERVER_transmit_context_create (client); + iim = (struct IndexInfoMessage*) buf; + pos = indexed_files; + while (NULL != pos) + { + fn = pos->filename; + slen = strlen (fn) + 1; + if (slen + sizeof (struct IndexInfoMessage) > + GNUNET_SERVER_MAX_MESSAGE_SIZE) + { + GNUNET_break (0); + break; + } + iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY); + iim->header.size = htons (slen + sizeof (struct IndexInfoMessage)); + iim->reserved = 0; + iim->file_id = pos->file_id; + memcpy (&iim[1], fn, slen); + GNUNET_SERVER_transmit_context_append_message (tc, + &iim->header); + pos = pos->next; + } + GNUNET_SERVER_transmit_context_append_data (tc, + NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END); + GNUNET_SERVER_transmit_context_run (tc, + GNUNET_TIME_UNIT_MINUTES); +} + + +/** + * Handle UNINDEX-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_unindex (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct UnindexMessage *um; + struct IndexInfo *pos; + struct IndexInfo *prev; + struct IndexInfo *next; + struct GNUNET_SERVER_TransmitContext *tc; + int found; + + um = (const struct UnindexMessage*) message; + found = GNUNET_NO; + prev = NULL; + pos = indexed_files; + while (NULL != pos) + { + next = pos->next; + if (0 == memcmp (&pos->file_id, + &um->file_id, + sizeof (GNUNET_HashCode))) + { + if (prev == NULL) + indexed_files = next; + else + prev->next = next; + GNUNET_free (pos); + found = GNUNET_YES; + } + else + { + prev = pos; + } + pos = next; + } +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Client requested unindexing of file `%s': %s\n", + GNUNET_h2s (&um->file_id), + found ? "found" : "not found"); +#endif + if (GNUNET_YES == found) + write_index_list (); + tc = GNUNET_SERVER_transmit_context_create (client); + GNUNET_SERVER_transmit_context_append_data (tc, + NULL, 0, + GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK); + GNUNET_SERVER_transmit_context_run (tc, + GNUNET_TIME_UNIT_MINUTES); +} + + + + +/** + * Continuation called from datastore's remove + * function. + * + * @param cls unused + * @param success did the deletion work? + * @param msg error message + */ +static void +remove_cont (void *cls, + int success, + const char *msg) +{ + struct GNUNET_DATASTORE_Handle *dsh = cls; + + if (GNUNET_OK != success) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to delete bogus block: %s\n"), + msg); + GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); +} + + +/** + * We've received an on-demand encoded block from the datastore. + * Attempt to do on-demand encoding and (if successful), call the + * continuation with the resulting block. On error, clean up and ask + * the datastore for more results. + * + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * @param cont function to call with the actual block + * @param cont_cls closure for cont + */ +void +GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key, + uint32_t size, + const void *data, + uint32_t type, + uint32_t priority, + uint32_t anonymity, + struct GNUNET_TIME_Absolute + expiration, uint64_t uid, + struct GNUNET_DATASTORE_Handle *dsh, + GNUNET_DATASTORE_Iterator cont, + void *cont_cls) +{ + const struct OnDemandBlock *odb; + GNUNET_HashCode nkey; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + GNUNET_HashCode query; + ssize_t nsize; + char ndata[DBLOCK_SIZE]; + char edata[DBLOCK_SIZE]; + const char *fn; + struct GNUNET_DISK_FileHandle *fh; + uint64_t off; + + if (size != sizeof (struct OnDemandBlock)) + { + GNUNET_break (0); + GNUNET_DATASTORE_remove (dsh, + key, + size, + data, + &remove_cont, + dsh, + GNUNET_TIME_UNIT_FOREVER_REL); + return; + } + odb = (const struct OnDemandBlock*) data; + off = GNUNET_ntohll (odb->offset); + fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm, + &odb->file_id); + fh = NULL; + if ( (NULL == fn) || + (NULL == (fh = GNUNET_DISK_file_open (fn, + GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE))) || + (off != + GNUNET_DISK_file_seek (fh, + off, + GNUNET_DISK_SEEK_SET)) || + (-1 == + (nsize = GNUNET_DISK_file_read (fh, + ndata, + sizeof (ndata)))) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Could not access indexed file `%s' at offset %llu: %s\n"), + GNUNET_h2s (&odb->file_id), + (unsigned long long) off, + STRERROR (errno)); + if (fh != NULL) + GNUNET_DISK_file_close (fh); + /* FIXME: if this happens often, we need + to remove the OnDemand block from the DS! */ + GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); + return; + } + GNUNET_DISK_file_close (fh); + GNUNET_CRYPTO_hash (ndata, + nsize, + &nkey); + GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv); + GNUNET_CRYPTO_aes_encrypt (ndata, + nsize, + &skey, + &iv, + edata); + GNUNET_CRYPTO_hash (edata, + nsize, + &query); + if (0 != memcmp (&query, + key, + sizeof (GNUNET_HashCode))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Indexed file `%s' changed at offset %llu\n"), + fn, + (unsigned long long) off); + /* FIXME: if this happens often, we need + to remove the OnDemand block from the DS! */ + GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); + return; + } + cont (cont_cls, + key, + nsize, + edata, + GNUNET_DATASTORE_BLOCKTYPE_DBLOCK, + priority, + anonymity, + expiration, + uid); +} + + +/** + * Task run during shutdown. + * + * @param cls unused + * @param tc unused + */ +static void +shutdown_task (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct IndexInfo *pos; + + GNUNET_CONTAINER_multihashmap_destroy (ifm); + ifm = NULL; + while (NULL != (pos = indexed_files)) + { + indexed_files = pos->next; + GNUNET_free (pos); + } + sched = NULL; + cfg = NULL; +} + + +/** + * Initialize the indexing submodule. + * + * @param s scheduler to use + * @param c configuration to use + */ +void +GNUNET_FS_init_indexing (struct GNUNET_SCHEDULER_Handle *s, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + sched = s; + cfg = c; + ifm = GNUNET_CONTAINER_multihashmap_create (128); + GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_UNIT_FOREVER_REL, + &shutdown_task, + NULL); + read_index_list (); +} + +/* end of gnunet-service-fs_indexing.c */ diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h new file mode 100644 index 000000000..dc6427234 --- /dev/null +++ b/src/fs/gnunet-service-fs_indexing.h @@ -0,0 +1,121 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_indexing.h + * @brief indexing for the file-sharing service + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_INDEXING_H +#define GNUNET_SERVICE_FS_INDEXING_H + +#include "gnunet_core_service.h" +#include "gnunet_datastore_service.h" +#include "gnunet_peer_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" + + +/** + * We've received an on-demand encoded block from the datastore. + * Attempt to do on-demand encoding and (if successful), call the + * continuation with the resulting block. On error, clean up and ask + * the datastore for more results. + * + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * @param dsh connection to the datastore (to ask for more) + * @param cont function to call with the actual block + * @param cont_cls closure for cont + */ +void +GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key, + uint32_t size, + const void *data, + uint32_t type, + uint32_t priority, + uint32_t anonymity, + struct GNUNET_TIME_Absolute + expiration, uint64_t uid, + struct GNUNET_DATASTORE_Handle *dsh, + GNUNET_DATASTORE_Iterator cont, + void *cont_cls); + +/** + * Handle INDEX_START-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_start (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + +/** + * Handle INDEX_LIST_GET-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_list_get (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + +/** + * Handle UNINDEX-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_unindex (void *cls, + struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + + + +/** + * Initialize the indexing submodule. + * + * @param s scheduler to use + * @param c configuration to use + */ +void +GNUNET_FS_init_indexing (struct GNUNET_SCHEDULER_Handle *s, + const struct GNUNET_CONFIGURATION_Handle *c); + + +#endif