2 This file is part of GNUnet
3 (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file datastore/datastore_api.c
23 * @brief Management for the datastore for files stored on a GNUnet node
24 * @author Christian Grothoff
27 #include "gnunet_datastore_service.h"
28 #include "datastore.h"
31 * Handle to the datastore service. Followed
32 * by 65536 bytes used for storing messages.
34 struct GNUNET_DATASTORE_Handle
40 const struct GNUNET_CONFIGURATION_Handle *cfg;
45 struct GNUNET_SCHEDULER_Handle *sched;
48 * Current connection to the datastore service.
50 struct GNUNET_CLIENT_Connection *client;
53 * Current response processor (NULL if we are not waiting for a
54 * response). The specific type depends on the kind of message we
60 * Closure for response_proc.
62 void *response_proc_cls;
65 * Timeout for the current operation.
67 struct GNUNET_TIME_Absolute timeout;
70 * Number of bytes in the message following
71 * this struct, 0 if we have no request pending.
79 * Connect to the datastore service.
81 * @param cfg configuration to use
82 * @param sched scheduler to use
83 * @return handle to use to access the service
85 struct GNUNET_DATASTORE_Handle *GNUNET_DATASTORE_connect (const struct
86 GNUNET_CONFIGURATION_Handle
89 GNUNET_SCHEDULER_Handle
92 struct GNUNET_CLIENT_Connection *c;
93 struct GNUNET_DATASTORE_Handle *h;
95 c = GNUNET_CLIENT_connect (sched, "datastore", cfg);
97 return NULL; /* oops */
98 h = GNUNET_malloc (sizeof(struct GNUNET_DATASTORE_Handle) +
99 GNUNET_SERVER_MAX_MESSAGE_SIZE);
108 * Transmit DROP message to datastore service.
111 transmit_drop (void *cls,
112 size_t size, void *buf)
114 struct GNUNET_DATASTORE_Handle *h = cls;
115 struct GNUNET_MessageHeader *hdr;
119 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
120 _("Failed to transmit request to drop database.\n"));
121 GNUNET_DATASTORE_disconnect (h, GNUNET_NO);
124 GNUNET_assert (size >= sizeof(struct GNUNET_MessageHeader));
126 hdr->size = htons(sizeof(struct GNUNET_MessageHeader));
127 hdr->type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_DROP);
128 GNUNET_DATASTORE_disconnect (h, GNUNET_NO);
129 return sizeof(struct GNUNET_MessageHeader);
134 * Disconnect from the datastore service (and free
135 * associated resources).
137 * @param h handle to the datastore
138 * @param drop set to GNUNET_YES to delete all data in datastore (!)
140 void GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
143 GNUNET_assert (0 == h->message_size);
144 GNUNET_assert (NULL == h->response_proc);
145 if ( (GNUNET_YES == drop) &&
146 (h->client != NULL) )
149 GNUNET_CLIENT_notify_transmit_ready (h->client,
150 sizeof(struct GNUNET_MessageHeader),
151 GNUNET_TIME_UNIT_MINUTES,
157 if (h->client != NULL)
158 GNUNET_CLIENT_disconnect (h->client);
164 * Type of a function to call when we receive a message
165 * from the service. This specific function is used
166 * to handle messages of type "struct StatusMessage".
169 * @param msg message received, NULL on timeout or fatal error
172 with_status_response_handler (void *cls,
174 GNUNET_MessageHeader * msg)
176 struct GNUNET_DATASTORE_Handle *h = cls;
177 GNUNET_DATASTORE_ContinuationWithStatus cont = h->response_proc;
178 const struct StatusMessage *sm;
184 h->response_proc = NULL;
185 GNUNET_CLIENT_disconnect (h->client);
186 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
187 cont (h->response_proc_cls,
189 _("Timeout trying to read response from datastore service"));
192 if ( (ntohs(msg->size) < sizeof(struct StatusMessage)) ||
193 (ntohs(msg->type) != GNUNET_MESSAGE_TYPE_DATASTORE_STATUS) )
196 h->response_proc = NULL;
197 GNUNET_CLIENT_disconnect (h->client);
198 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
199 cont (h->response_proc_cls,
201 _("Error reading response from datastore service"));
204 sm = (const struct StatusMessage*) msg;
205 status = ntohl(sm->status);
207 if (ntohs(msg->size) > sizeof(struct StatusMessage))
209 emsg = (const char*) &sm[1];
210 if (emsg[ntohs(msg->size) - sizeof(struct StatusMessage) - 1] != '\0')
213 emsg = _("Invalid error message received from datastore service");
216 if ( (status == GNUNET_SYSERR) &&
220 emsg = _("Invalid error message received from datastore service");
222 h->response_proc = NULL;
224 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
225 "Received status %d/%s\n",
229 cont (h->response_proc_cls,
236 * Transmit message to datastore service and then
237 * read a status message.
239 * @param cls closure with handle to datastore
240 * @param size number of bytes we can transmit at most
241 * @param buf where to write transmission, NULL on
243 * @return number of bytes copied to buf
246 transmit_get_status (void *cls,
250 struct GNUNET_DATASTORE_Handle *h = cls;
251 GNUNET_DATASTORE_ContinuationWithStatus cont = h->response_proc;
257 h->response_proc = NULL;
258 cont (h->response_proc_cls,
260 _("Error transmitting message to datastore service."));
263 msize = h->message_size;
264 GNUNET_assert (msize <= size);
265 memcpy (buf, &h[1], msize);
267 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
268 "Transmitted %u byte message to datastore service, now waiting for status.\n",
272 GNUNET_CLIENT_receive (h->client,
273 &with_status_response_handler,
275 GNUNET_TIME_absolute_get_remaining (h->timeout));
281 * Helper function that will initiate the
282 * transmission of a message to the datastore
283 * service. The message must already be prepared
284 * and stored in the buffer at the end of the
285 * handle. The message must be of a type that
286 * expects a "StatusMessage" in response.
288 * @param h handle to the service with prepared message
289 * @param cont function to call with result
290 * @param cont_cls closure
291 * @param timeout timeout for the operation
294 transmit_for_status (struct GNUNET_DATASTORE_Handle *h,
295 GNUNET_DATASTORE_ContinuationWithStatus cont,
297 struct GNUNET_TIME_Relative timeout)
299 const struct GNUNET_MessageHeader *hdr;
302 GNUNET_assert (cont != NULL);
303 hdr = (const struct GNUNET_MessageHeader*) &h[1];
304 msize = ntohs(hdr->size);
306 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
307 "Transmitting %u byte message of type %u to datastore service\n",
311 GNUNET_assert (h->response_proc == NULL);
312 h->response_proc = cont;
313 h->response_proc_cls = cont_cls;
314 h->timeout = GNUNET_TIME_relative_to_absolute (timeout);
315 h->message_size = msize;
316 if (NULL == GNUNET_CLIENT_notify_transmit_ready (h->client,
319 &transmit_get_status,
323 h->response_proc = NULL;
327 _("Not ready to transmit request to datastore service"));
333 * Store an item in the datastore. If the item is already present,
334 * the priorities are summed up and the higher expiration time and
335 * lower anonymity level is used.
337 * @param h handle to the datastore
338 * @param rid reservation ID to use (from "reserve"); use 0 if no
339 * prior reservation was made
340 * @param key key for the value
341 * @param size number of bytes in data
342 * @param data content stored
343 * @param type type of the content
344 * @param priority priority of the content
345 * @param anonymity anonymity-level for the content
346 * @param expiration expiration time for the content
347 * @param timeout timeout for the operation
348 * @param cont continuation to call when done
349 * @param cont_cls closure for cont
352 GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
354 const GNUNET_HashCode * key,
360 struct GNUNET_TIME_Absolute expiration,
361 struct GNUNET_TIME_Relative timeout,
362 GNUNET_DATASTORE_ContinuationWithStatus cont,
365 struct DataMessage *dm;
369 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
370 "Asked to put %u bytes of data under key `%s'\n",
374 msize = sizeof(struct DataMessage) + size;
375 GNUNET_assert (msize <= GNUNET_SERVER_MAX_MESSAGE_SIZE);
376 dm = (struct DataMessage*) &h[1];
377 dm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_PUT);
378 dm->header.size = htons(msize);
379 dm->rid = htonl(rid);
380 dm->size = htonl(size);
381 dm->type = htonl(type);
382 dm->priority = htonl(priority);
383 dm->anonymity = htonl(anonymity);
384 dm->uid = GNUNET_htonll(0);
385 dm->expiration = GNUNET_TIME_absolute_hton(expiration);
387 memcpy (&dm[1], data, size);
388 transmit_for_status (h, cont, cont_cls, timeout);
393 * Reserve space in the datastore. This function should be used
394 * to avoid "out of space" failures during a longer sequence of "put"
395 * operations (for example, when a file is being inserted).
397 * @param h handle to the datastore
398 * @param amount how much space (in bytes) should be reserved (for content only)
399 * @param entries how many entries will be created (to calculate per-entry overhead)
400 * @param cont continuation to call when done; "success" will be set to
401 * a positive reservation value if space could be reserved.
402 * @param cont_cls closure for cont
403 * @param timeout how long to wait at most for a response
406 GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h,
409 GNUNET_DATASTORE_ContinuationWithStatus cont,
411 struct GNUNET_TIME_Relative timeout)
413 struct ReserveMessage *rm;
415 rm = (struct ReserveMessage*) &h[1];
416 rm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE);
417 rm->header.size = htons(sizeof (struct ReserveMessage));
418 rm->entries = htonl(entries);
419 rm->amount = GNUNET_htonll(amount);
420 transmit_for_status (h, cont, cont_cls, timeout);
425 * Signal that all of the data for which a reservation was made has
426 * been stored and that whatever excess space might have been reserved
427 * can now be released.
429 * @param h handle to the datastore
430 * @param rid reservation ID (value of "success" in original continuation
431 * from the "reserve" function).
432 * @param cont continuation to call when done
433 * @param cont_cls closure for cont
434 * @param timeout how long to wait at most for a response
437 GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
439 GNUNET_DATASTORE_ContinuationWithStatus cont,
441 struct GNUNET_TIME_Relative timeout)
443 struct ReleaseReserveMessage *rrm;
445 rrm = (struct ReleaseReserveMessage*) &h[1];
446 rrm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE);
447 rrm->header.size = htons(sizeof (struct ReleaseReserveMessage));
448 rrm->rid = htonl(rid);
449 transmit_for_status (h, cont, cont_cls, timeout);
454 * Update a value in the datastore.
456 * @param h handle to the datastore
457 * @param uid identifier for the value
458 * @param priority how much to increase the priority of the value
459 * @param expiration new expiration value should be MAX of existing and this argument
460 * @param cont continuation to call when done
461 * @param cont_cls closure for cont
462 * @param timeout how long to wait at most for a response
465 GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
466 unsigned long long uid,
468 struct GNUNET_TIME_Absolute expiration,
469 GNUNET_DATASTORE_ContinuationWithStatus cont,
471 struct GNUNET_TIME_Relative timeout)
473 struct UpdateMessage *um;
475 um = (struct UpdateMessage*) &h[1];
476 um->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE);
477 um->header.size = htons(sizeof (struct UpdateMessage));
478 um->priority = htonl(priority);
479 um->expiration = GNUNET_TIME_absolute_hton(expiration);
480 um->uid = GNUNET_htonll(uid);
481 transmit_for_status (h, cont, cont_cls, timeout);
488 * Type of a function to call when we receive a message
489 * from the service. This specific function is used
490 * to handle messages of type "struct DataMessage".
493 * @param msg message received, NULL on timeout or fatal error
496 with_result_response_handler (void *cls,
498 GNUNET_MessageHeader * msg)
500 struct GNUNET_DATASTORE_Handle *h = cls;
501 GNUNET_DATASTORE_Iterator cont = h->response_proc;
502 const struct DataMessage *dm;
507 h->response_proc = NULL;
508 GNUNET_CLIENT_disconnect (h->client);
509 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
510 cont (h->response_proc_cls,
511 NULL, 0, NULL, 0, 0, 0,
512 GNUNET_TIME_UNIT_ZERO_ABS, 0);
515 if (ntohs(msg->type) == GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END)
517 GNUNET_break (ntohs(msg->size) == sizeof(struct GNUNET_MessageHeader));
518 h->response_proc = NULL;
520 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
521 "Received end of result set\n");
523 cont (h->response_proc_cls,
524 NULL, 0, NULL, 0, 0, 0,
525 GNUNET_TIME_UNIT_ZERO_ABS, 0);
528 if ( (ntohs(msg->size) < sizeof(struct DataMessage)) ||
529 (ntohs(msg->type) != GNUNET_MESSAGE_TYPE_DATASTORE_DATA) )
532 GNUNET_CLIENT_disconnect (h->client);
533 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
534 h->response_proc = NULL;
535 cont (h->response_proc_cls,
536 NULL, 0, NULL, 0, 0, 0,
537 GNUNET_TIME_UNIT_ZERO_ABS, 0);
540 dm = (const struct DataMessage*) msg;
541 msize = ntohl(dm->size);
542 if (ntohs(msg->size) != msize + sizeof(struct DataMessage))
545 GNUNET_CLIENT_disconnect (h->client);
546 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
547 h->response_proc = NULL;
548 cont (h->response_proc_cls,
549 NULL, 0, NULL, 0, 0, 0,
550 GNUNET_TIME_UNIT_ZERO_ABS, 0);
554 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
555 "Received result %llu with type %u and size %u with key %s\n",
556 (unsigned long long) GNUNET_ntohll(dm->uid),
559 GNUNET_h2s(&dm->key));
561 cont (h->response_proc_cls,
567 ntohl(dm->anonymity),
568 GNUNET_TIME_absolute_ntoh(dm->expiration),
569 GNUNET_ntohll(dm->uid));
574 * Transmit message to datastore service and then
575 * read a result message.
577 * @param cls closure with handle to datastore
578 * @param size number of bytes we can transmit at most
579 * @param buf where to write transmission, NULL on
581 * @return number of bytes copied to buf
584 transmit_get_result (void *cls,
588 struct GNUNET_DATASTORE_Handle *h = cls;
589 GNUNET_DATASTORE_Iterator cont = h->response_proc;
594 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
595 _("Error transmitting message to datastore service.\n"));
596 h->response_proc = NULL;
598 cont (h->response_proc_cls,
599 NULL, 0, NULL, 0, 0, 0,
600 GNUNET_TIME_UNIT_ZERO_ABS, 0);
603 msize = h->message_size;
604 GNUNET_assert (msize <= size);
605 memcpy (buf, &h[1], msize);
607 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
608 "Transmitted %u byte message to datastore service, now waiting for result.\n",
612 GNUNET_CLIENT_receive (h->client,
613 &with_result_response_handler,
615 GNUNET_TIME_absolute_get_remaining (h->timeout));
622 * Function called to trigger obtaining the next result
623 * from the datastore.
625 * @param h handle to the datastore
626 * @param more GNUNET_YES to get moxre results, GNUNET_NO to abort
627 * iteration (with a final call to "iter" with key/data == NULL).
630 GNUNET_DATASTORE_get_next (struct GNUNET_DATASTORE_Handle *h,
633 GNUNET_DATASTORE_Iterator cont;
635 if (GNUNET_YES == more)
637 GNUNET_CLIENT_receive (h->client,
638 &with_result_response_handler,
640 GNUNET_TIME_absolute_get_remaining (h->timeout));
643 cont = h->response_proc;
644 h->response_proc = NULL;
645 GNUNET_CLIENT_disconnect (h->client);
646 h->client = GNUNET_CLIENT_connect (h->sched, "datastore", h->cfg);
647 cont (h->response_proc_cls,
648 NULL, 0, NULL, 0, 0, 0,
649 GNUNET_TIME_UNIT_ZERO_ABS, 0);
654 * Helper function that will initiate the
655 * transmission of a message to the datastore
656 * service. The message must already be prepared
657 * and stored in the buffer at the end of the
658 * handle. The message must be of a type that
659 * expects a "DataMessage" in response.
661 * @param h handle to the service with prepared message
662 * @param cont function to call with result
663 * @param cont_cls closure
664 * @param timeout timeout for the operation
667 transmit_for_result (struct GNUNET_DATASTORE_Handle *h,
668 GNUNET_DATASTORE_Iterator cont,
670 struct GNUNET_TIME_Relative timeout)
672 const struct GNUNET_MessageHeader *hdr;
675 GNUNET_assert (cont != NULL);
676 hdr = (const struct GNUNET_MessageHeader*) &h[1];
677 msize = ntohs(hdr->size);
679 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
680 "Transmitting %u byte message of type %u to datastore service\n",
684 GNUNET_assert (h->response_proc == NULL);
685 h->response_proc = cont;
686 h->response_proc_cls = cont_cls;
687 h->timeout = GNUNET_TIME_relative_to_absolute (timeout);
688 h->message_size = msize;
689 if (NULL == GNUNET_CLIENT_notify_transmit_ready (h->client,
692 &transmit_get_result,
696 h->response_proc = NULL;
698 cont (h->response_proc_cls,
699 NULL, 0, NULL, 0, 0, 0,
700 GNUNET_TIME_UNIT_ZERO_ABS, 0);
706 * Iterate over the results for a particular key
709 * @param h handle to the datastore
710 * @param key maybe NULL (to match all entries)
711 * @param type desired type, 0 for any
712 * @param iter function to call on each matching value;
713 * will be called once with a NULL value at the end
714 * @param iter_cls closure for iter
715 * @param timeout how long to wait at most for a response
718 GNUNET_DATASTORE_get (struct GNUNET_DATASTORE_Handle *h,
719 const GNUNET_HashCode * key,
721 GNUNET_DATASTORE_Iterator iter, void *iter_cls,
722 struct GNUNET_TIME_Relative timeout)
724 struct GetMessage *gm;
727 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
728 "Asked to look for data under key `%s'\n",
731 gm = (struct GetMessage*) &h[1];
732 gm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET);
733 gm->type = htonl(type);
736 gm->header.size = htons(sizeof (struct GetMessage));
741 gm->header.size = htons(sizeof (struct GetMessage) - sizeof(GNUNET_HashCode));
743 transmit_for_result (h, iter, iter_cls, timeout);
748 * Get a random value from the datastore.
750 * @param h handle to the datastore
751 * @param iter function to call on a random value; it
752 * will be called exactly once; if no values
753 * are available, the value will be NULL.
754 * @param iter_cls closure for iter
755 * @param timeout how long to wait at most for a response
758 GNUNET_DATASTORE_get_random (struct GNUNET_DATASTORE_Handle *h,
759 GNUNET_DATASTORE_Iterator iter, void *iter_cls,
760 struct GNUNET_TIME_Relative timeout)
762 struct GNUNET_MessageHeader *m;
764 m = (struct GNUNET_MessageHeader*) &h[1];
765 m->type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_GET_RANDOM);
766 m->size = htons(sizeof (struct GNUNET_MessageHeader));
767 transmit_for_result (h, iter, iter_cls, timeout);
772 * Explicitly remove some content from the database.
774 * @param h handle to the datastore
775 * @param key key for the value
776 * @param size number of bytes in data
777 * @param data content stored
778 * @param cont continuation to call when done
779 * @param cont_cls closure for cont
780 * @param timeout how long to wait at most for a response
783 GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
784 const GNUNET_HashCode * key,
785 uint32_t size, const void *data,
786 GNUNET_DATASTORE_ContinuationWithStatus cont,
788 struct GNUNET_TIME_Relative timeout)
790 struct DataMessage *dm;
793 msize = sizeof(struct DataMessage) + size;
794 GNUNET_assert (msize <= GNUNET_SERVER_MAX_MESSAGE_SIZE);
795 dm = (struct DataMessage*) &h[1];
796 dm->header.type = htons(GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
797 dm->header.size = htons(msize);
799 dm->size = htonl(size);
801 dm->priority = htonl(0);
802 dm->anonymity = htonl(0);
803 dm->uid = GNUNET_htonll(0);
804 dm->expiration = GNUNET_TIME_absolute_hton(GNUNET_TIME_UNIT_ZERO_ABS);
806 memcpy (&dm[1], data, size);
807 transmit_for_status (h, cont, cont_cls, timeout);
811 /* end of datastore_api.c */