2 This file is part of GNUnet.
3 (C) 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file fs/gnunet-service-fs.c
23 * @brief program that provides the file-sharing service
24 * @author Christian Grothoff
27 * - INDEX_START handling
29 * - bloomfilter support (GET, CS-request with BF, etc.)
33 #include "gnunet_protocols.h"
34 #include "gnunet_core_service.h"
35 #include "gnunet_datastore_service.h"
36 #include "gnunet_util_lib.h"
40 * Our connection to the datastore.
42 static struct GNUNET_DATASTORE_Handle *dsh;
47 static struct GNUNET_SCHEDULER_Handle *sched;
52 const struct GNUNET_CONFIGURATION_Handle *cfg;
56 * In-memory information about indexed files (also available
63 * This is a linked list.
65 struct IndexInfo *next;
68 * Name of the indexed file. Memory allocated
69 * at the end of this struct (do not free).
74 * Context for tranmitting confirmation to client,
75 * NULL if we've done this already.
77 struct GNUNET_SERVER_TransmitContext *tc;
80 * Hash of the contents of the file.
82 GNUNET_HashCode file_id;
88 * Linked list of indexed files.
90 static struct IndexInfo *indexed_files;
94 * Write the current index information list to disk.
97 write_index_list (void)
103 * Read index information from disk.
106 read_index_list (void)
113 * We've validated the hash of the file we're about to
114 * index. Signal success to the client and update
115 * our internal data structures.
117 * @param ii the index info entry for the request
120 signal_index_ok (struct IndexInfo *ii)
122 ii->next = indexed_files;
125 GNUNET_SERVER_transmit_context_append (ii->tc,
127 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
128 GNUNET_SERVER_transmit_context_run (ii->tc,
129 GNUNET_TIME_UNIT_MINUTES);
136 * Function called once the hash computation over an
137 * indexed file has completed.
139 * @param cls closure, our publishing context
140 * @param res resulting hash, NULL on error
143 hash_for_index_val (void *cls,
144 const GNUNET_HashCode *
147 struct IndexInfo *ii;
149 if ( (res == NULL) ||
152 sizeof(GNUNET_HashCode))) )
154 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
155 _("Hash mismatch trying to index file `%s'\n"),
157 GNUNET_SERVER_transmit_context_append (ii->tc,
159 GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
160 GNUNET_SERVER_transmit_context_run (ii->tc,
161 GNUNET_TIME_UNIT_MINUTES);
165 signal_index_ok (ii);
170 * Handle INDEX_START-message.
173 * @param client identification of the client
174 * @param message the actual message
177 handle_index_start (void *cls,
178 struct GNUNET_SERVER_Client *client,
179 const struct GNUNET_MessageHeader *message)
181 const struct IndexStartMessage *ism;
184 struct IndexInfo *ii;
191 msize = ntohs(message->size);
192 if ( (msize <= sizeof (struct IndexStartMessage)) ||
193 ( ((const char *)message)[msize-1] != '\0') )
196 GNUNET_SERVER_receive_done (client,
201 fn = (const char*) &ism[1];
202 dev = ntohl (ism->device);
203 ino = GNUNET_ntohll (ism->inode);
204 ism = (const struct IndexStartMessage*) message;
205 slen = strlen (fn) + 1;
206 ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
207 ii->filename = (const char*) &ii[1];
208 memcpy (&ii[1], fn, slen);
209 ii->file_id = ism->file_id;
210 ii->tc = GNUNET_SERVER_transmit_context_create (client);
213 (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
219 /* fast validation OK! */
220 signal_index_ok (ii);
223 /* slow validation, need to hash full file (again) */
224 GNUNET_CRYPTO_hash_file (sched,
225 GNUNET_SCHEDULER_PRIORITY_IDLE,
235 * Handle INDEX_LIST_GET-message.
238 * @param client identification of the client
239 * @param message the actual message
242 handle_index_list_get (void *cls,
243 struct GNUNET_SERVER_Client *client,
244 const struct GNUNET_MessageHeader *message)
246 struct GNUNET_SERVER_TransmitContext *tc;
247 struct IndexInfoMessage *iim;
248 char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
251 struct GNUNET_MessageHeader *msg;
252 struct IndexInfo *pos;
254 tc = GNUNET_SERVER_transmit_context_create (client);
255 iim = (struct IndexInfoMessage*) buf;
261 iim->file_id = pos->file_id;
263 slen = strlen (fn) + 1;
264 if (slen + sizeof (struct IndexInfoMessage) >
265 GNUNET_SERVER_MAX_MESSAGE_SIZE)
270 memcpy (&iim[1], fn, slen);
271 GNUNET_SERVER_transmit_context_append
274 sizeof (struct IndexInfoMessage)
275 - sizeof (struct GNUNET_MessageHeader) + slen,
276 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
279 GNUNET_SERVER_transmit_context_append (tc,
281 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
282 GNUNET_SERVER_transmit_context_run (tc,
283 GNUNET_TIME_UNIT_MINUTES);
288 * Handle UNINDEX-message.
291 * @param client identification of the client
292 * @param message the actual message
295 handle_unindex (void *cls,
296 struct GNUNET_SERVER_Client *client,
297 const struct GNUNET_MessageHeader *message)
299 const struct UnindexMessage *um;
300 struct GNUNET_SERVER_TransmitContext *tc;
302 um = (const struct UnindexMessage*) message;
304 tc = GNUNET_SERVER_transmit_context_create (client);
305 GNUNET_SERVER_transmit_context_append (tc,
307 GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
308 GNUNET_SERVER_transmit_context_run (tc,
309 GNUNET_TIME_UNIT_MINUTES);
314 * Signature of a function that is called whenever a datastore
315 * request can be processed (or an entry put on the queue times out).
318 * @param ok GNUNET_OK if DS is ready, GNUNET_SYSERR on timeout
320 typedef void (*RequestFunction)(void *cls,
325 * Doubly-linked list of our requests for the datastore.
327 struct DatastoreRequestQueue
331 * This is a doubly-linked list.
333 struct DatastoreRequestQueue *next;
336 * This is a doubly-linked list.
338 struct DatastoreRequestQueue *prev;
341 * Function to call (will issue the request).
351 * When should this request time-out because we don't care anymore?
353 struct GNUNET_TIME_Absolute timeout;
356 * ID of task used for signaling timeout.
358 GNUNET_SCHEDULER_TaskIdentifier task;
364 * Head of request queue for the datastore, sorted by timeout.
366 static struct DatastoreRequestQueue *drq_head;
369 * Tail of request queue for the datastore.
371 static struct DatastoreRequestQueue *drq_tail;
375 * Run the next DS request in our
376 * queue, we're done with the current one.
381 struct DatastoreRequestQueue *e;
383 while (NULL != (e = drq_head))
385 if (0 != GNUNET_TIME_absolute_get_remaining (e->timeout).value)
387 if (e->task != GNUNET_SCHEDULER_NO_TASK)
388 GNUNET_SCHEDULER_cancel (sched, e->task);
389 GNUNET_CONTAINER_DLL_remove (drq_head, drq_tail, e);
390 e->req (e->req_cls, GNUNET_NO);
395 if (e->task != GNUNET_SCHEDULER_NO_TASK)
396 GNUNET_SCHEDULER_cancel (sched, e->task);
397 e->task = GNUNET_SCHEDULER_NO_TASK;
398 e->req (e->req_cls, GNUNET_YES);
399 GNUNET_CONTAINER_DLL_remove (drq_head, drq_tail, e);
405 * A datastore request had to be timed out.
407 * @param cls closure (of type "struct DatastoreRequestQueue*")
408 * @param tc task context, unused
411 timeout_ds_request (void *cls,
412 const struct GNUNET_SCHEDULER_TaskContext *tc)
414 struct DatastoreRequestQueue *e = cls;
416 e->task = GNUNET_SCHEDULER_NO_TASK;
417 GNUNET_CONTAINER_DLL_remove (drq_head, drq_tail, e);
418 e->req (e->req_cls, GNUNET_NO);
424 * Queue a request for the datastore.
426 * @param deadline by when the request should run
427 * @param fun function to call once the request can be run
428 * @param fun_cls closure for fun
430 static struct DatastoreRequestQueue *
431 queue_ds_request (struct GNUNET_TIME_Relative deadline,
435 struct DatastoreRequestQueue *e;
436 struct DatastoreRequestQueue *bef;
438 if (drq_head == NULL)
440 /* no other requests pending, run immediately */
441 fun (fun_cls, GNUNET_OK);
444 e = GNUNET_malloc (sizeof (struct DatastoreRequestQueue));
445 e->timeout = GNUNET_TIME_relative_to_absolute (deadline);
447 e->req_cls = fun_cls;
448 if (deadline.value == GNUNET_TIME_UNIT_FOREVER_REL.value)
450 /* local request, highest prio, put at head of queue
451 regardless of deadline */
457 while ( (NULL != bef) &&
458 (e->timeout.value < bef->timeout.value) )
461 GNUNET_CONTAINER_DLL_insert_after (drq_head, drq_tail, bef, e);
462 if (deadline.value == GNUNET_TIME_UNIT_FOREVER_REL.value)
464 e->task = GNUNET_SCHEDULER_add_delayed (sched,
466 GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
467 GNUNET_SCHEDULER_NO_TASK,
476 * Closure for processing START_SEARCH messages from a client.
478 struct LocalGetContext
482 * This is a doubly-linked list.
484 struct LocalGetContext *next;
487 * This is a doubly-linked list.
489 struct LocalGetContext *prev;
492 * Client that initiated the search.
494 struct GNUNET_SERVER_Client *client;
497 * Array of results that we've already received
500 GNUNET_HashCode *results;
503 * Bloomfilter over all results (for fast query construction);
504 * NULL if we don't have any results.
506 struct GNUNET_CONTAINER_BloomFilter *results_bf;
509 * DS request associated with this operation.
511 struct DatastoreRequestQueue *req;
514 * Current result message to transmit to client (or NULL).
516 struct ContentMessage *result;
519 * Type of the content that we're looking for.
525 * Desired anonymity level.
527 uint32_t anonymity_level;
530 * Number of results actually stored in the results array.
532 unsigned int results_used;
535 * Size of the results array in memory.
537 unsigned int results_size;
540 * If the request is for a DBLOCK or IBLOCK, this is the identity of
541 * the peer that is known to have a response. Set to all-zeros if
542 * such a target is not known (note that even if OUR anonymity
543 * level is >0 we may happen to know the responder's identity;
544 * nevertheless, we should probably not use it for a DHT-lookup
545 * or similar blunt actions in order to avoid exposing ourselves).
547 * If the request is for an SBLOCK, this is the identity of the
548 * pseudonym to which the SBLOCK belongs.
550 * If the request is for a KBLOCK, "target" must be all zeros.
552 GNUNET_HashCode target;
555 * Hash of the keyword (aka query) for KBLOCKs; Hash of
556 * the CHK-encoded block for DBLOCKS and IBLOCKS (aka query)
557 * and hash of the identifier XORed with the target for
558 * SBLOCKS (aka query).
560 GNUNET_HashCode query;
566 * Head of doubly-linked LGC list.
568 static struct LocalGetContext *lgc_head;
571 * Tail of doubly-linked LGC list.
573 static struct LocalGetContext *lgc_tail;
577 * Free the state associated with a local get context.
579 * @param lgc the lgc to free
582 local_get_context_free (struct LocalGetContext *lgc)
584 GNUNET_CONTAINER_DLL_remove (lgc_head, lgc_tail, lgc);
585 GNUNET_SERVER_client_drop (lgc->client);
586 GNUNET_free_non_null (lgc->results);
587 if (lgc->results_bf != NULL)
588 GNUNET_CONTAINER_bloomfilter_free (lgc->results_bf);
589 if (lgc->req != NULL)
591 if (lgc->req->task != GNUNET_SCHEDULER_NO_TASK)
592 GNUNET_SCHEDULER_cancel (sched, lgc->req->task);
593 GNUNET_CONTAINER_DLL_remove (lgc_head, lgc_tail, lgc);
594 GNUNET_free (lgc->req);
601 * We're able to transmit the next (local) result to the client.
602 * Do it and ask the datastore for more. Or, on error, tell
603 * the datastore to stop giving us more.
605 * @param cls our closure (struct LocalGetContext)
606 * @param max maximum number of bytes we can transmit
607 * @param buf where to copy our message
608 * @return number of bytes copied to buf
611 transmit_local_result (void *cls,
615 struct LocalGetContext *lgc = cls;
621 GNUNET_free (lgc->result);
623 GNUNET_DATASTORE_get_next (dsh, GNUNET_NO);
626 msize = ntohs (lgc->result->header.size);
627 GNUNET_assert (max >= msize);
628 memcpy (buf, lgc->result, msize);
629 GNUNET_free (lgc->result);
631 GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
637 * We're processing (local) results for a search request
638 * from a (local) client. Pass applicable results to the
639 * client and if we are done either clean up (operation
640 * complete) or switch to P2P search (more results possible).
642 * @param cls our closure (struct LocalGetContext)
643 * @param key key for the content
644 * @param size number of bytes in data
645 * @param data content stored
646 * @param type type of the content
647 * @param priority priority of the content
648 * @param anonymity anonymity-level for the content
649 * @param expiration expiration time for the content
650 * @param uid unique identifier for the datum;
651 * maybe 0 if no unique identifier is available
654 process_local_get_result (void *cls,
655 const GNUNET_HashCode * key,
661 struct GNUNET_TIME_Absolute
665 struct LocalGetContext *lgc = cls;
670 /* no further results from datastore; continue
671 processing further requests from the client and
672 allow the next task to use the datastore; also,
673 switch to P2P requests or clean up our state. */
675 GNUNET_SERVER_receive_done (lgc->client,
677 if ( (lgc->results_used == 0) ||
678 (lgc->type == GNUNET_DATASTORE_BLOCKTYPE_KBLOCK) ||
679 (lgc->type == GNUNET_DATASTORE_BLOCKTYPE_SBLOCK) ||
680 (lgc->type == GNUNET_DATASTORE_BLOCKTYPE_SKBLOCK) )
682 // FIXME: initiate P2P search
685 /* got all possible results, clean up! */
686 local_get_context_free (lgc);
689 if (lgc->results_used == lgc->results_size)
691 GNUNET_array_grow (lgc->results,
693 lgc->results_size * 2 + 2);
694 if ( (lgc->type != GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) ||
695 (lgc->type != GNUNET_DATASTORE_BLOCKTYPE_IBLOCK) )
697 // FIXME: possibly grow/create BF!
700 GNUNET_CRYPTO_hash (data,
702 &lgc->results[lgc->results_used++]);
703 if ( (lgc->type != GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) ||
704 (lgc->type != GNUNET_DATASTORE_BLOCKTYPE_IBLOCK) )
706 // FIXME: add result to BF!
708 msize = size + sizeof (struct ContentMessage);
709 GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE);
710 lgc->result = GNUNET_malloc (msize);
711 lgc->result->header.size = htons (msize);
712 lgc->result->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CONTENT);
713 lgc->result->type = htonl (type);
714 lgc->result->expiration = GNUNET_TIME_absolute_hton (expiration);
715 memcpy (&lgc->result[1],
718 GNUNET_SERVER_notify_transmit_ready (lgc->client,
720 GNUNET_TIME_UNIT_FOREVER_REL,
721 &transmit_local_result,
727 * We're processing a search request from a local
728 * client. Now it is our turn to query the datastore.
730 * @param cls our closure (struct LocalGetContext)
734 transmit_local_get (void *cls,
735 const struct GNUNET_SCHEDULER_TaskContext *tc)
737 struct LocalGetContext *lgc = cls;
739 GNUNET_DATASTORE_get (dsh,
742 &process_local_get_result,
744 GNUNET_TIME_UNIT_FOREVER_REL);
749 * We're processing a search request from a local
750 * client. Now it is our turn to query the datastore.
752 * @param cls our closure (struct LocalGetContext)
753 * @param ok did we succeed to queue for datastore access, should always be GNUNET_OK
756 transmit_local_get_ready (void *cls,
759 struct LocalGetContext *lgc = cls;
761 GNUNET_assert (GNUNET_OK == ok);
762 GNUNET_SCHEDULER_add_continuation (sched,
766 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
771 * Handle START_SEARCH-message (search request from client).
774 * @param client identification of the client
775 * @param message the actual message
778 handle_start_search (void *cls,
779 struct GNUNET_SERVER_Client *client,
780 const struct GNUNET_MessageHeader *message)
782 const struct SearchMessage *sm;
783 struct LocalGetContext *lgc;
785 sm = (const struct SearchMessage*) message;
786 GNUNET_SERVER_client_keep (client);
787 lgc = GNUNET_malloc (sizeof (struct LocalGetContext));
788 lgc->client = client;
789 lgc->type = ntohl (sm->type);
790 lgc->anonymity_level = ntohl (sm->anonymity_level);
791 lgc->target = sm->target;
792 lgc->query = sm->query;
793 GNUNET_CONTAINER_DLL_insert (lgc_head, lgc_tail, lgc);
794 lgc->req = queue_ds_request (GNUNET_TIME_UNIT_FOREVER_REL,
795 &transmit_local_get_ready,
801 * List of handlers for the messages understood by this
804 static struct GNUNET_SERVER_MessageHandler handlers[] = {
805 {&handle_index_start, NULL,
806 GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0},
807 {&handle_index_list_get, NULL,
808 GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, sizeof(struct GNUNET_MessageHeader) },
809 {&handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX,
810 sizeof (struct UnindexMessage) },
811 {&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH,
812 sizeof (struct SearchMessage) },
818 * A client disconnected. Remove all of its pending queries.
820 * @param cls closure, NULL
821 * @param client identification of the client
824 handle_client_disconnect (void *cls,
825 struct GNUNET_SERVER_Client
828 struct LocalGetContext *lgc;
831 while ( (NULL != lgc) &&
832 (lgc->client != client) )
835 return; /* not one of our clients */
836 local_get_context_free (lgc);
841 * Task run during shutdown.
847 shutdown_task (void *cls,
848 const struct GNUNET_SCHEDULER_TaskContext *tc)
850 GNUNET_DATASTORE_disconnect (dsh,
857 * Process fs requests.
860 * @param sched scheduler to use
861 * @param server the initialized server
862 * @param cfg configuration to use
866 struct GNUNET_SCHEDULER_Handle *s,
867 struct GNUNET_SERVER_Handle *server,
868 const struct GNUNET_CONFIGURATION_Handle *c)
873 dsh = GNUNET_DATASTORE_connect (cfg,
877 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
878 _("Failed to connect to datastore service.\n"));
881 GNUNET_SERVER_disconnect_notify (server,
882 &handle_client_disconnect,
884 GNUNET_SERVER_add_handlers (server, handlers);
885 // FIXME: also register with core to handle P2P messages!
886 GNUNET_SCHEDULER_add_delayed (sched,
888 GNUNET_SCHEDULER_PRIORITY_IDLE,
889 GNUNET_SCHEDULER_NO_TASK,
890 GNUNET_TIME_UNIT_FOREVER_REL,
897 * The main function for the fs service.
899 * @param argc number of arguments from the command line
900 * @param argv command line arguments
901 * @return 0 ok, 1 on error
904 main (int argc, char *const *argv)
907 GNUNET_SERVICE_run (argc,
909 "fs", &run, NULL, NULL, NULL)) ? 0 : 1;
912 /* end of gnunet-service-fs.c */