#include "gnunet_protocols.h"
#include "gnunet_signatures.h"
#include "gnunet_util_lib.h"
+#include "gnunet-service-fs_indexing.h"
#include "fs.h"
#define DEBUG_FS GNUNET_NO
-/**
- * In-memory information about indexed files (also available
- * on-disk).
- */
-struct IndexInfo
-{
-
- /**
- * This is a linked list.
- */
- struct IndexInfo *next;
-
- /**
- * Name of the indexed file. Memory allocated
- * at the end of this struct (do not free).
- */
- const char *filename;
-
- /**
- * Context for transmitting confirmation to client,
- * NULL if we've done this already.
- */
- struct GNUNET_SERVER_TransmitContext *tc;
-
- /**
- * Hash of the contents of the file.
- */
- GNUNET_HashCode file_id;
-
-};
-
/**
* Signature of a function that is called whenever a datastore
/**
- * Information we keep for each pending reply.
+ * Information we keep for each pending reply. The
+ * actual message follows at the end of this struct.
*/
-struct PendingReply
+struct PendingMessage
{
/**
* This is a linked list.
*/
- struct PendingReply *next;
+ struct PendingMessage *next;
/**
* Size of the reply; actual reply message follows
* at the end of this struct.
*/
size_t msize;
+
+ /**
+ * How important is this message for us?
+ */
+ uint32_t priority;
};
* (typically non-null only if we have a pending transmission
* request with the client or the respective peer).
*/
- struct PendingReply *replies_pending;
+ struct PendingMessage *replies_pending;
/**
* Pending transmission request with the core service for the target
*/
struct GNUNET_TIME_Relative avg_delay;
+ /**
+ * Handle for an active request for transmission to this
+ * peer, or NULL.
+ */
+ struct GNUNET_CORE_PeerRequestHandle *prh;
+
+ /**
+ * Messages (replies, queries, content migration) we would like to
+ * send to this peer in the near future. Sorted by priority.
+ */
+ struct PendingMessage *pending_messages;
+
/**
* Average priority of successful replies. Calculated
* as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n
GNUNET_PEER_Id pid;
/**
- * Number of requests we have currently pending
- * with this peer (that is, requests that were
- * transmitted so recently that we would not retransmit
- * them right now).
+ * Number of requests we have currently pending with this peer (that
+ * is, requests that were transmitted so recently that we would not
+ * retransmit them right now).
*/
unsigned int pending_requests;
*/
static struct DatastoreRequestQueue *drq_tail;
-/**
- * Linked list of indexed files.
- */
-static struct IndexInfo *indexed_files;
-
-/**
- * Maps hash over content of indexed files to the respective filename.
- * The filenames are pointers into the indexed_files linked list and
- * do not need to be freed.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *ifm;
-
/**
* Map of query hash codes to requests.
*/
static uint64_t max_pending_requests = 32;
-/**
- * Write the current index information list to disk.
- */
-static void
-write_index_list ()
-{
- struct GNUNET_BIO_WriteHandle *wh;
- char *fn;
- struct IndexInfo *pos;
-
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_filename (cfg,
- "FS",
- "INDEXDB",
- &fn))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
- _("Configuration option `%s' in section `%s' missing.\n"),
- "INDEXDB",
- "FS");
- return;
- }
- wh = GNUNET_BIO_write_open (fn);
- if (NULL == wh)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
- _("Could not open `%s'.\n"),
- fn);
- GNUNET_free (fn);
- return;
- }
- pos = indexed_files;
- while (pos != NULL)
- {
- if ( (GNUNET_OK !=
- GNUNET_BIO_write (wh,
- &pos->file_id,
- sizeof (GNUNET_HashCode))) ||
- (GNUNET_OK !=
- GNUNET_BIO_write_string (wh,
- pos->filename)) )
- break;
- pos = pos->next;
- }
- if (GNUNET_OK !=
- GNUNET_BIO_write_close (wh))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
- _("Error writing `%s'.\n"),
- fn);
- GNUNET_free (fn);
- return;
- }
- GNUNET_free (fn);
-}
-
-
-/**
- * Read index information from disk.
- */
-static void
-read_index_list ()
-{
- struct GNUNET_BIO_ReadHandle *rh;
- char *fn;
- struct IndexInfo *pos;
- char *fname;
- GNUNET_HashCode hc;
- size_t slen;
- char *emsg;
-
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_filename (cfg,
- "FS",
- "INDEXDB",
- &fn))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
- _("Configuration option `%s' in section `%s' missing.\n"),
- "INDEXDB",
- "FS");
- return;
- }
- if (GNUNET_NO == GNUNET_DISK_file_test (fn))
- {
- /* no index info yet */
- GNUNET_free (fn);
- return;
- }
- rh = GNUNET_BIO_read_open (fn);
- if (NULL == rh)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
- _("Could not open `%s'.\n"),
- fn);
- GNUNET_free (fn);
- return;
- }
-
- while ( (GNUNET_OK ==
- GNUNET_BIO_read (rh,
- "Hash of indexed file",
- &hc,
- sizeof (GNUNET_HashCode))) &&
- (GNUNET_OK ==
- GNUNET_BIO_read_string (rh,
- "Name of indexed file",
- &fname,
- 1024 * 16)) )
- {
- slen = strlen (fname) + 1;
- pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
- pos->file_id = hc;
- pos->filename = (const char *) &pos[1];
- memcpy (&pos[1], fname, slen);
- if (GNUNET_SYSERR ==
- GNUNET_CONTAINER_multihashmap_put (ifm,
- &hc,
- (void*) pos->filename,
- GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
- {
- GNUNET_free (pos);
- }
- else
- {
- pos->next = indexed_files;
- indexed_files = pos;
- }
- GNUNET_free (fname);
- }
- if (GNUNET_OK !=
- GNUNET_BIO_read_close (rh, &emsg))
- GNUNET_free (emsg);
- GNUNET_free (fn);
-}
-
-
-/**
- * We've validated the hash of the file we're about to
- * index. Signal success to the client and update
- * our internal data structures.
- *
- * @param ii the index info entry for the request
- */
-static void
-signal_index_ok (struct IndexInfo *ii)
-{
- if (GNUNET_SYSERR ==
- GNUNET_CONTAINER_multihashmap_put (ifm,
- &ii->file_id,
- (void*) ii->filename,
- GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"),
- ii->filename,
- (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
- &ii->file_id));
- GNUNET_SERVER_transmit_context_append_data (ii->tc,
- NULL, 0,
- GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
- GNUNET_SERVER_transmit_context_run (ii->tc,
- GNUNET_TIME_UNIT_MINUTES);
- GNUNET_free (ii);
- return;
- }
- ii->next = indexed_files;
- indexed_files = ii;
- write_index_list ();
- GNUNET_SERVER_transmit_context_append_data (ii->tc,
- NULL, 0,
- GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
- GNUNET_SERVER_transmit_context_run (ii->tc,
- GNUNET_TIME_UNIT_MINUTES);
- ii->tc = NULL;
-}
-
-
-/**
- * Function called once the hash computation over an
- * indexed file has completed.
- *
- * @param cls closure, our publishing context
- * @param res resulting hash, NULL on error
- */
-static void
-hash_for_index_val (void *cls,
- const GNUNET_HashCode *
- res)
-{
- struct IndexInfo *ii = cls;
-
- if ( (res == NULL) ||
- (0 != memcmp (res,
- &ii->file_id,
- sizeof(GNUNET_HashCode))) )
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
- ii->filename,
- GNUNET_h2s (res));
-#if DEBUG_FS
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Wanted `%s'\n",
- GNUNET_h2s (&ii->file_id));
-#endif
- GNUNET_SERVER_transmit_context_append_data (ii->tc,
- NULL, 0,
- GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
- GNUNET_SERVER_transmit_context_run (ii->tc,
- GNUNET_TIME_UNIT_MINUTES);
- GNUNET_free (ii);
- return;
- }
- signal_index_ok (ii);
-}
-
-
-/**
- * Handle INDEX_START-message.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_index_start (void *cls,
- struct GNUNET_SERVER_Client *client,
- const struct GNUNET_MessageHeader *message)
-{
- const struct IndexStartMessage *ism;
- const char *fn;
- uint16_t msize;
- struct IndexInfo *ii;
- size_t slen;
- uint32_t dev;
- uint64_t ino;
- uint32_t mydev;
- uint64_t myino;
-
- msize = ntohs(message->size);
- if ( (msize <= sizeof (struct IndexStartMessage)) ||
- ( ((const char *)message)[msize-1] != '\0') )
- {
- GNUNET_break (0);
- GNUNET_SERVER_receive_done (client,
- GNUNET_SYSERR);
- return;
- }
- ism = (const struct IndexStartMessage*) message;
- fn = (const char*) &ism[1];
- dev = ntohl (ism->device);
- ino = GNUNET_ntohll (ism->inode);
- ism = (const struct IndexStartMessage*) message;
- slen = strlen (fn) + 1;
- ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
- ii->filename = (const char*) &ii[1];
- memcpy (&ii[1], fn, slen);
- ii->file_id = ism->file_id;
- ii->tc = GNUNET_SERVER_transmit_context_create (client);
- if ( ( (dev != 0) ||
- (ino != 0) ) &&
- (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
- &mydev,
- &myino)) &&
- ( (dev == mydev) &&
- (ino == myino) ) )
- {
- /* fast validation OK! */
- signal_index_ok (ii);
- return;
- }
-#if DEBUG_FS
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
- (unsigned long long) ino,
- (unsigned long long) myino,
- (unsigned int) dev,
- (unsigned int) mydev);
-#endif
- /* slow validation, need to hash full file (again) */
- GNUNET_CRYPTO_hash_file (sched,
- GNUNET_SCHEDULER_PRIORITY_IDLE,
- fn,
- HASHING_BLOCKSIZE,
- &hash_for_index_val,
- ii);
-}
-
-
-/**
- * Handle INDEX_LIST_GET-message.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_index_list_get (void *cls,
- struct GNUNET_SERVER_Client *client,
- const struct GNUNET_MessageHeader *message)
-{
- struct GNUNET_SERVER_TransmitContext *tc;
- struct IndexInfoMessage *iim;
- char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
- size_t slen;
- const char *fn;
- struct IndexInfo *pos;
-
- tc = GNUNET_SERVER_transmit_context_create (client);
- iim = (struct IndexInfoMessage*) buf;
- pos = indexed_files;
- while (NULL != pos)
- {
- fn = pos->filename;
- slen = strlen (fn) + 1;
- if (slen + sizeof (struct IndexInfoMessage) >
- GNUNET_SERVER_MAX_MESSAGE_SIZE)
- {
- GNUNET_break (0);
- break;
- }
- iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
- iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
- iim->reserved = 0;
- iim->file_id = pos->file_id;
- memcpy (&iim[1], fn, slen);
- GNUNET_SERVER_transmit_context_append_message (tc,
- &iim->header);
- pos = pos->next;
- }
- GNUNET_SERVER_transmit_context_append_data (tc,
- NULL, 0,
- GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
- GNUNET_SERVER_transmit_context_run (tc,
- GNUNET_TIME_UNIT_MINUTES);
-}
-/**
- * Handle UNINDEX-message.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_unindex (void *cls,
- struct GNUNET_SERVER_Client *client,
- const struct GNUNET_MessageHeader *message)
-{
- const struct UnindexMessage *um;
- struct IndexInfo *pos;
- struct IndexInfo *prev;
- struct IndexInfo *next;
- struct GNUNET_SERVER_TransmitContext *tc;
- int found;
-
- um = (const struct UnindexMessage*) message;
- found = GNUNET_NO;
- prev = NULL;
- pos = indexed_files;
- while (NULL != pos)
- {
- next = pos->next;
- if (0 == memcmp (&pos->file_id,
- &um->file_id,
- sizeof (GNUNET_HashCode)))
- {
- if (prev == NULL)
- indexed_files = next;
- else
- prev->next = next;
- GNUNET_free (pos);
- found = GNUNET_YES;
- }
- else
- {
- prev = pos;
- }
- pos = next;
- }
-#if DEBUG_FS
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Client requested unindexing of file `%s': %s\n",
- GNUNET_h2s (&um->file_id),
- found ? "found" : "not found");
-#endif
- if (GNUNET_YES == found)
- write_index_list ();
- tc = GNUNET_SERVER_transmit_context_create (client);
- GNUNET_SERVER_transmit_context_append_data (tc,
- NULL, 0,
- GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
- GNUNET_SERVER_transmit_context_run (tc,
- GNUNET_TIME_UNIT_MINUTES);
-}
/**
}
-/**
- * Continuation called from datastore's remove
- * function.
- *
- * @param cls unused
- * @param success did the deletion work?
- * @param msg error message
- */
-static void
-remove_cont (void *cls,
- int success,
- const char *msg)
-{
- if (GNUNET_OK != success)
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _("Failed to delete bogus block: %s\n"),
- msg);
- GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
-}
-
-
/**
* Mingle hash with the mingle_number to
* produce different bits.
}
-/**
- * We've received an on-demand encoded block
- * from the datastore. Attempt to do on-demand
- * encoding and (if successful), call the
- * continuation with the resulting block. On
- * error, clean up and ask the datastore for
- * more results.
- *
- * @param key key for the content
- * @param size number of bytes in data
- * @param data content stored
- * @param type type of the content
- * @param priority priority of the content
- * @param anonymity anonymity-level for the content
- * @param expiration expiration time for the content
- * @param uid unique identifier for the datum;
- * maybe 0 if no unique identifier is available
- * @param cont function to call with the actual block
- * @param cont_cls closure for cont
- */
-static void
-handle_on_demand_block (const GNUNET_HashCode * key,
- uint32_t size,
- const void *data,
- uint32_t type,
- uint32_t priority,
- uint32_t anonymity,
- struct GNUNET_TIME_Absolute
- expiration, uint64_t uid,
- GNUNET_DATASTORE_Iterator cont,
- void *cont_cls)
-{
- const struct OnDemandBlock *odb;
- GNUNET_HashCode nkey;
- struct GNUNET_CRYPTO_AesSessionKey skey;
- struct GNUNET_CRYPTO_AesInitializationVector iv;
- GNUNET_HashCode query;
- ssize_t nsize;
- char ndata[DBLOCK_SIZE];
- char edata[DBLOCK_SIZE];
- const char *fn;
- struct GNUNET_DISK_FileHandle *fh;
- uint64_t off;
-
- if (size != sizeof (struct OnDemandBlock))
- {
- GNUNET_break (0);
- GNUNET_DATASTORE_remove (dsh,
- key,
- size,
- data,
- &remove_cont,
- NULL,
- GNUNET_TIME_UNIT_FOREVER_REL);
- return;
- }
- odb = (const struct OnDemandBlock*) data;
- off = GNUNET_ntohll (odb->offset);
- fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
- &odb->file_id);
- fh = NULL;
- if ( (NULL == fn) ||
- (NULL == (fh = GNUNET_DISK_file_open (fn,
- GNUNET_DISK_OPEN_READ,
- GNUNET_DISK_PERM_NONE))) ||
- (off !=
- GNUNET_DISK_file_seek (fh,
- off,
- GNUNET_DISK_SEEK_SET)) ||
- (-1 ==
- (nsize = GNUNET_DISK_file_read (fh,
- ndata,
- sizeof (ndata)))) )
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _("Could not access indexed file `%s' at offset %llu: %s\n"),
- GNUNET_h2s (&odb->file_id),
- (unsigned long long) off,
- STRERROR (errno));
- if (fh != NULL)
- GNUNET_DISK_file_close (fh);
- /* FIXME: if this happens often, we need
- to remove the OnDemand block from the DS! */
- GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
- return;
- }
- GNUNET_DISK_file_close (fh);
- GNUNET_CRYPTO_hash (ndata,
- nsize,
- &nkey);
- GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
- GNUNET_CRYPTO_aes_encrypt (ndata,
- nsize,
- &skey,
- &iv,
- edata);
- GNUNET_CRYPTO_hash (edata,
- nsize,
- &query);
- if (0 != memcmp (&query,
- key,
- sizeof (GNUNET_HashCode)))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _("Indexed file `%s' changed at offset %llu\n"),
- fn,
- (unsigned long long) off);
- /* FIXME: if this happens often, we need
- to remove the OnDemand block from the DS! */
- GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
- return;
- }
- cont (cont_cls,
- key,
- nsize,
- edata,
- GNUNET_DATASTORE_BLOCKTYPE_DBLOCK,
- priority,
- anonymity,
- expiration,
- uid);
-}
-
-
/**
* How many bytes should a bloomfilter be if we have already seen
* entry_count responses? Note that BLOOMFILTER_K gives us the number
"Received on-demand block for `%s' from local datastore, fetching data.\n",
GNUNET_h2s (&lgc->query));
#endif
- handle_on_demand_block (key, size, data, type, priority,
- anonymity, expiration, uid,
- &process_local_get_result,
- lgc);
+ GNUNET_FS_handle_on_demand_block (key, size, data, type, priority,
+ anonymity, expiration, uid,
+ dsh,
+ &process_local_get_result,
+ lgc);
return;
}
if ( (type != lgc->type) &&
* service.
*/
static struct GNUNET_SERVER_MessageHandler handlers[] = {
- {&handle_index_start, NULL,
+ {&GNUNET_FS_handle_index_start, NULL,
GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0},
- {&handle_index_list_get, NULL,
+ {&GNUNET_FS_handle_index_list_get, NULL,
GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, sizeof(struct GNUNET_MessageHeader) },
- {&handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX,
+ {&GNUNET_FS_handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX,
sizeof (struct UnindexMessage) },
{&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH,
0 },
static void
destroy_pending_request (struct PendingRequest *pr)
{
- struct PendingReply *reply;
+ struct PendingMessage *reply;
struct ClientList *cl;
GNUNET_CONTAINER_multihashmap_remove (requests_by_query,
shutdown_task (void *cls,
const struct GNUNET_SCHEDULER_TaskContext *tc)
{
- struct IndexInfo *pos;
-
if (NULL != core)
{
GNUNET_CORE_disconnect (core);
// (or do we get disconnect notifications?)
GNUNET_CONTAINER_multihashmap_destroy (connected_peers);
connected_peers = NULL;
- GNUNET_CONTAINER_multihashmap_destroy (ifm);
- ifm = NULL;
- while (NULL != (pos = indexed_files))
- {
- indexed_files = pos->next;
- GNUNET_free (pos);
- }
}
}
if (type == GNUNET_DATASTORE_BLOCKTYPE_ONDEMAND)
{
- handle_on_demand_block (key, size, data, type, priority,
- anonymity, expiration, uid,
- &process_p2p_get_result,
- pgc);
+ GNUNET_FS_handle_on_demand_block (key, size, data, type, priority,
+ anonymity, expiration, uid, dsh,
+ &process_p2p_get_result,
+ pgc);
return;
}
/* check for duplicates */
{
struct PendingRequest *pr = cls;
char *cbuf = buf;
- struct PendingReply *reply;
+ struct PendingMessage *reply;
size_t ret;
ret = 0;
struct ProcessReplyClosure *prq = cls;
struct PendingRequest *pr = value;
struct PendingRequest *eer;
- struct PendingReply *reply;
+ struct PendingMessage *reply;
struct PutMessage *pm;
struct ContentMessage *cm;
GNUNET_HashCode chash;
if (pr->client == NULL)
{
msize = sizeof (struct ContentMessage) + prq->size;
- reply = GNUNET_malloc (msize + sizeof (struct PendingReply));
+ reply = GNUNET_malloc (msize + sizeof (struct PendingMessage));
reply->msize = msize;
cm = (struct ContentMessage*) &reply[1];
cm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CONTENT);
else
{
msize = sizeof (struct PutMessage) + prq->size;
- reply = GNUNET_malloc (msize + sizeof (struct PendingReply));
+ reply = GNUNET_malloc (msize + sizeof (struct PendingMessage));
reply->msize = msize;
reply->next = pr->replies_pending;
pm = (struct PutMessage*) &reply[1];
sched = s;
cfg = c;
- ifm = GNUNET_CONTAINER_multihashmap_create (128);
+
requests_by_query = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config
requests_by_peer = GNUNET_CONTAINER_multihashmap_create (128); // FIXME: get size from config
connected_peers = GNUNET_CONTAINER_multihashmap_create (64);
requests_by_expiration = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
- read_index_list ();
+ GNUNET_FS_init_indexing (sched, cfg);
dsh = GNUNET_DATASTORE_connect (cfg,
sched);
core = GNUNET_CORE_connect (sched,
--- /dev/null
+/*
+ This file is part of GNUnet.
+ (C) 2009, 2010 Christian Grothoff (and other contributing authors)
+
+ GNUnet is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 2, or (at your
+ option) any later version.
+
+ GNUnet is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GNUnet; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ Boston, MA 02111-1307, USA.
+*/
+
+/**
+ * @file fs/gnunet-service-fs_indexing.c
+ * @brief program that provides indexing functions of the file-sharing service
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - indexed files/blocks not removed on errors
+ */
+#include "platform.h"
+#include <float.h>
+#include "gnunet_core_service.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_peer_lib.h"
+#include "gnunet_protocols.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+#include "gnunet-service-fs_indexing.h"
+#include "fs.h"
+
+#define DEBUG_FS GNUNET_NO
+
+
+/**
+ * In-memory information about indexed files (also available
+ * on-disk).
+ */
+struct IndexInfo
+{
+
+ /**
+ * This is a linked list.
+ */
+ struct IndexInfo *next;
+
+ /**
+ * Name of the indexed file. Memory allocated
+ * at the end of this struct (do not free).
+ */
+ const char *filename;
+
+ /**
+ * Context for transmitting confirmation to client,
+ * NULL if we've done this already.
+ */
+ struct GNUNET_SERVER_TransmitContext *tc;
+
+ /**
+ * Hash of the contents of the file.
+ */
+ GNUNET_HashCode file_id;
+
+};
+
+
+/**
+ * Linked list of indexed files.
+ */
+static struct IndexInfo *indexed_files;
+
+/**
+ * Maps hash over content of indexed files to the respective filename.
+ * The filenames are pointers into the indexed_files linked list and
+ * do not need to be freed.
+ */
+static struct GNUNET_CONTAINER_MultiHashMap *ifm;
+
+/**
+ * Our scheduler.
+ */
+static struct GNUNET_SCHEDULER_Handle *sched;
+
+/**
+ * Our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+
+/**
+ * Write the current index information list to disk.
+ */
+static void
+write_index_list ()
+{
+ struct GNUNET_BIO_WriteHandle *wh;
+ char *fn;
+ struct IndexInfo *pos;
+
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_filename (cfg,
+ "FS",
+ "INDEXDB",
+ &fn))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+ _("Configuration option `%s' in section `%s' missing.\n"),
+ "INDEXDB",
+ "FS");
+ return;
+ }
+ wh = GNUNET_BIO_write_open (fn);
+ if (NULL == wh)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+ _("Could not open `%s'.\n"),
+ fn);
+ GNUNET_free (fn);
+ return;
+ }
+ pos = indexed_files;
+ while (pos != NULL)
+ {
+ if ( (GNUNET_OK !=
+ GNUNET_BIO_write (wh,
+ &pos->file_id,
+ sizeof (GNUNET_HashCode))) ||
+ (GNUNET_OK !=
+ GNUNET_BIO_write_string (wh,
+ pos->filename)) )
+ break;
+ pos = pos->next;
+ }
+ if (GNUNET_OK !=
+ GNUNET_BIO_write_close (wh))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+ _("Error writing `%s'.\n"),
+ fn);
+ GNUNET_free (fn);
+ return;
+ }
+ GNUNET_free (fn);
+}
+
+
+/**
+ * Read index information from disk.
+ */
+static void
+read_index_list ()
+{
+ struct GNUNET_BIO_ReadHandle *rh;
+ char *fn;
+ struct IndexInfo *pos;
+ char *fname;
+ GNUNET_HashCode hc;
+ size_t slen;
+ char *emsg;
+
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_filename (cfg,
+ "FS",
+ "INDEXDB",
+ &fn))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+ _("Configuration option `%s' in section `%s' missing.\n"),
+ "INDEXDB",
+ "FS");
+ return;
+ }
+ if (GNUNET_NO == GNUNET_DISK_file_test (fn))
+ {
+ /* no index info yet */
+ GNUNET_free (fn);
+ return;
+ }
+ rh = GNUNET_BIO_read_open (fn);
+ if (NULL == rh)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+ _("Could not open `%s'.\n"),
+ fn);
+ GNUNET_free (fn);
+ return;
+ }
+
+ while ( (GNUNET_OK ==
+ GNUNET_BIO_read (rh,
+ "Hash of indexed file",
+ &hc,
+ sizeof (GNUNET_HashCode))) &&
+ (GNUNET_OK ==
+ GNUNET_BIO_read_string (rh,
+ "Name of indexed file",
+ &fname,
+ 1024 * 16)) )
+ {
+ slen = strlen (fname) + 1;
+ pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
+ pos->file_id = hc;
+ pos->filename = (const char *) &pos[1];
+ memcpy (&pos[1], fname, slen);
+ if (GNUNET_SYSERR ==
+ GNUNET_CONTAINER_multihashmap_put (ifm,
+ &hc,
+ (void*) pos->filename,
+ GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+ {
+ GNUNET_free (pos);
+ }
+ else
+ {
+ pos->next = indexed_files;
+ indexed_files = pos;
+ }
+ GNUNET_free (fname);
+ }
+ if (GNUNET_OK !=
+ GNUNET_BIO_read_close (rh, &emsg))
+ GNUNET_free (emsg);
+ GNUNET_free (fn);
+}
+
+
+/**
+ * We've validated the hash of the file we're about to index. Signal
+ * success to the client and update our internal data structures.
+ *
+ * @param ii the index info entry for the request
+ */
+static void
+signal_index_ok (struct IndexInfo *ii)
+{
+ if (GNUNET_SYSERR ==
+ GNUNET_CONTAINER_multihashmap_put (ifm,
+ &ii->file_id,
+ (void*) ii->filename,
+ GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("Index request received for file `%s' is indexed as `%s'. Permitting anyway.\n"),
+ ii->filename,
+ (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
+ &ii->file_id));
+ GNUNET_SERVER_transmit_context_append_data (ii->tc,
+ NULL, 0,
+ GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
+ GNUNET_SERVER_transmit_context_run (ii->tc,
+ GNUNET_TIME_UNIT_MINUTES);
+ GNUNET_free (ii);
+ return;
+ }
+ ii->next = indexed_files;
+ indexed_files = ii;
+ write_index_list ();
+ GNUNET_SERVER_transmit_context_append_data (ii->tc,
+ NULL, 0,
+ GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
+ GNUNET_SERVER_transmit_context_run (ii->tc,
+ GNUNET_TIME_UNIT_MINUTES);
+ ii->tc = NULL;
+}
+
+
+/**
+ * Function called once the hash computation over an
+ * indexed file has completed.
+ *
+ * @param cls closure, our publishing context
+ * @param res resulting hash, NULL on error
+ */
+static void
+hash_for_index_val (void *cls,
+ const GNUNET_HashCode *
+ res)
+{
+ struct IndexInfo *ii = cls;
+
+ if ( (res == NULL) ||
+ (0 != memcmp (res,
+ &ii->file_id,
+ sizeof(GNUNET_HashCode))) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
+ ii->filename,
+ GNUNET_h2s (res));
+#if DEBUG_FS
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Wanted `%s'\n",
+ GNUNET_h2s (&ii->file_id));
+#endif
+ GNUNET_SERVER_transmit_context_append_data (ii->tc,
+ NULL, 0,
+ GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
+ GNUNET_SERVER_transmit_context_run (ii->tc,
+ GNUNET_TIME_UNIT_MINUTES);
+ GNUNET_free (ii);
+ return;
+ }
+ signal_index_ok (ii);
+}
+
+
+/**
+ * Handle INDEX_START-message.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ */
+void
+GNUNET_FS_handle_index_start (void *cls,
+ struct GNUNET_SERVER_Client *client,
+ const struct GNUNET_MessageHeader *message)
+{
+ const struct IndexStartMessage *ism;
+ const char *fn;
+ uint16_t msize;
+ struct IndexInfo *ii;
+ size_t slen;
+ uint32_t dev;
+ uint64_t ino;
+ uint32_t mydev;
+ uint64_t myino;
+
+ msize = ntohs(message->size);
+ if ( (msize <= sizeof (struct IndexStartMessage)) ||
+ ( ((const char *)message)[msize-1] != '\0') )
+ {
+ GNUNET_break (0);
+ GNUNET_SERVER_receive_done (client,
+ GNUNET_SYSERR);
+ return;
+ }
+ ism = (const struct IndexStartMessage*) message;
+ fn = (const char*) &ism[1];
+ dev = ntohl (ism->device);
+ ino = GNUNET_ntohll (ism->inode);
+ ism = (const struct IndexStartMessage*) message;
+ slen = strlen (fn) + 1;
+ ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
+ ii->filename = (const char*) &ii[1];
+ memcpy (&ii[1], fn, slen);
+ ii->file_id = ism->file_id;
+ ii->tc = GNUNET_SERVER_transmit_context_create (client);
+ if ( ( (dev != 0) ||
+ (ino != 0) ) &&
+ (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
+ &mydev,
+ &myino)) &&
+ ( (dev == mydev) &&
+ (ino == myino) ) )
+ {
+ /* fast validation OK! */
+ signal_index_ok (ii);
+ return;
+ }
+#if DEBUG_FS
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
+ (unsigned long long) ino,
+ (unsigned long long) myino,
+ (unsigned int) dev,
+ (unsigned int) mydev);
+#endif
+ /* slow validation, need to hash full file (again) */
+ GNUNET_CRYPTO_hash_file (sched,
+ GNUNET_SCHEDULER_PRIORITY_IDLE,
+ fn,
+ HASHING_BLOCKSIZE,
+ &hash_for_index_val,
+ ii);
+}
+
+
+/**
+ * Handle INDEX_LIST_GET-message.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ */
+void
+GNUNET_FS_handle_index_list_get (void *cls,
+ struct GNUNET_SERVER_Client *client,
+ const struct GNUNET_MessageHeader *message)
+{
+ struct GNUNET_SERVER_TransmitContext *tc;
+ struct IndexInfoMessage *iim;
+ char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
+ size_t slen;
+ const char *fn;
+ struct IndexInfo *pos;
+
+ tc = GNUNET_SERVER_transmit_context_create (client);
+ iim = (struct IndexInfoMessage*) buf;
+ pos = indexed_files;
+ while (NULL != pos)
+ {
+ fn = pos->filename;
+ slen = strlen (fn) + 1;
+ if (slen + sizeof (struct IndexInfoMessage) >
+ GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ {
+ GNUNET_break (0);
+ break;
+ }
+ iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
+ iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
+ iim->reserved = 0;
+ iim->file_id = pos->file_id;
+ memcpy (&iim[1], fn, slen);
+ GNUNET_SERVER_transmit_context_append_message (tc,
+ &iim->header);
+ pos = pos->next;
+ }
+ GNUNET_SERVER_transmit_context_append_data (tc,
+ NULL, 0,
+ GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
+ GNUNET_SERVER_transmit_context_run (tc,
+ GNUNET_TIME_UNIT_MINUTES);
+}
+
+
+/**
+ * Handle UNINDEX-message.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ */
+void
+GNUNET_FS_handle_unindex (void *cls,
+ struct GNUNET_SERVER_Client *client,
+ const struct GNUNET_MessageHeader *message)
+{
+ const struct UnindexMessage *um;
+ struct IndexInfo *pos;
+ struct IndexInfo *prev;
+ struct IndexInfo *next;
+ struct GNUNET_SERVER_TransmitContext *tc;
+ int found;
+
+ um = (const struct UnindexMessage*) message;
+ found = GNUNET_NO;
+ prev = NULL;
+ pos = indexed_files;
+ while (NULL != pos)
+ {
+ next = pos->next;
+ if (0 == memcmp (&pos->file_id,
+ &um->file_id,
+ sizeof (GNUNET_HashCode)))
+ {
+ if (prev == NULL)
+ indexed_files = next;
+ else
+ prev->next = next;
+ GNUNET_free (pos);
+ found = GNUNET_YES;
+ }
+ else
+ {
+ prev = pos;
+ }
+ pos = next;
+ }
+#if DEBUG_FS
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Client requested unindexing of file `%s': %s\n",
+ GNUNET_h2s (&um->file_id),
+ found ? "found" : "not found");
+#endif
+ if (GNUNET_YES == found)
+ write_index_list ();
+ tc = GNUNET_SERVER_transmit_context_create (client);
+ GNUNET_SERVER_transmit_context_append_data (tc,
+ NULL, 0,
+ GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
+ GNUNET_SERVER_transmit_context_run (tc,
+ GNUNET_TIME_UNIT_MINUTES);
+}
+
+
+
+
+/**
+ * Continuation called from datastore's remove
+ * function.
+ *
+ * @param cls unused
+ * @param success did the deletion work?
+ * @param msg error message
+ */
+static void
+remove_cont (void *cls,
+ int success,
+ const char *msg)
+{
+ struct GNUNET_DATASTORE_Handle *dsh = cls;
+
+ if (GNUNET_OK != success)
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("Failed to delete bogus block: %s\n"),
+ msg);
+ GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
+}
+
+
+/**
+ * We've received an on-demand encoded block from the datastore.
+ * Attempt to do on-demand encoding and (if successful), call the
+ * continuation with the resulting block. On error, clean up and ask
+ * the datastore for more results.
+ *
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ * maybe 0 if no unique identifier is available
+ * @param cont function to call with the actual block
+ * @param cont_cls closure for cont
+ */
+void
+GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
+ uint32_t size,
+ const void *data,
+ uint32_t type,
+ uint32_t priority,
+ uint32_t anonymity,
+ struct GNUNET_TIME_Absolute
+ expiration, uint64_t uid,
+ struct GNUNET_DATASTORE_Handle *dsh,
+ GNUNET_DATASTORE_Iterator cont,
+ void *cont_cls)
+{
+ const struct OnDemandBlock *odb;
+ GNUNET_HashCode nkey;
+ struct GNUNET_CRYPTO_AesSessionKey skey;
+ struct GNUNET_CRYPTO_AesInitializationVector iv;
+ GNUNET_HashCode query;
+ ssize_t nsize;
+ char ndata[DBLOCK_SIZE];
+ char edata[DBLOCK_SIZE];
+ const char *fn;
+ struct GNUNET_DISK_FileHandle *fh;
+ uint64_t off;
+
+ if (size != sizeof (struct OnDemandBlock))
+ {
+ GNUNET_break (0);
+ GNUNET_DATASTORE_remove (dsh,
+ key,
+ size,
+ data,
+ &remove_cont,
+ dsh,
+ GNUNET_TIME_UNIT_FOREVER_REL);
+ return;
+ }
+ odb = (const struct OnDemandBlock*) data;
+ off = GNUNET_ntohll (odb->offset);
+ fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
+ &odb->file_id);
+ fh = NULL;
+ if ( (NULL == fn) ||
+ (NULL == (fh = GNUNET_DISK_file_open (fn,
+ GNUNET_DISK_OPEN_READ,
+ GNUNET_DISK_PERM_NONE))) ||
+ (off !=
+ GNUNET_DISK_file_seek (fh,
+ off,
+ GNUNET_DISK_SEEK_SET)) ||
+ (-1 ==
+ (nsize = GNUNET_DISK_file_read (fh,
+ ndata,
+ sizeof (ndata)))) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("Could not access indexed file `%s' at offset %llu: %s\n"),
+ GNUNET_h2s (&odb->file_id),
+ (unsigned long long) off,
+ STRERROR (errno));
+ if (fh != NULL)
+ GNUNET_DISK_file_close (fh);
+ /* FIXME: if this happens often, we need
+ to remove the OnDemand block from the DS! */
+ GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
+ return;
+ }
+ GNUNET_DISK_file_close (fh);
+ GNUNET_CRYPTO_hash (ndata,
+ nsize,
+ &nkey);
+ GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
+ GNUNET_CRYPTO_aes_encrypt (ndata,
+ nsize,
+ &skey,
+ &iv,
+ edata);
+ GNUNET_CRYPTO_hash (edata,
+ nsize,
+ &query);
+ if (0 != memcmp (&query,
+ key,
+ sizeof (GNUNET_HashCode)))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("Indexed file `%s' changed at offset %llu\n"),
+ fn,
+ (unsigned long long) off);
+ /* FIXME: if this happens often, we need
+ to remove the OnDemand block from the DS! */
+ GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
+ return;
+ }
+ cont (cont_cls,
+ key,
+ nsize,
+ edata,
+ GNUNET_DATASTORE_BLOCKTYPE_DBLOCK,
+ priority,
+ anonymity,
+ expiration,
+ uid);
+}
+
+
+/**
+ * Task run during shutdown.
+ *
+ * @param cls unused
+ * @param tc unused
+ */
+static void
+shutdown_task (void *cls,
+ const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+ struct IndexInfo *pos;
+
+ GNUNET_CONTAINER_multihashmap_destroy (ifm);
+ ifm = NULL;
+ while (NULL != (pos = indexed_files))
+ {
+ indexed_files = pos->next;
+ GNUNET_free (pos);
+ }
+ sched = NULL;
+ cfg = NULL;
+}
+
+
+/**
+ * Initialize the indexing submodule.
+ *
+ * @param s scheduler to use
+ * @param c configuration to use
+ */
+void
+GNUNET_FS_init_indexing (struct GNUNET_SCHEDULER_Handle *s,
+ const struct GNUNET_CONFIGURATION_Handle *c)
+{
+ sched = s;
+ cfg = c;
+ ifm = GNUNET_CONTAINER_multihashmap_create (128);
+ GNUNET_SCHEDULER_add_delayed (sched,
+ GNUNET_TIME_UNIT_FOREVER_REL,
+ &shutdown_task,
+ NULL);
+ read_index_list ();
+}
+
+/* end of gnunet-service-fs_indexing.c */
projects / oweals / gnunet.git / commitdiff