2 This file is part of GNUnet
3 Copyright (C) 2004-2013, 2016 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
22 * @file datastore/datastore_api.c
23 * @brief Management for the datastore for files stored on a GNUnet node. Implements
24 * a priority queue for requests
25 * @author Christian Grothoff
28 #include "gnunet_arm_service.h"
29 #include "gnunet_constants.h"
30 #include "gnunet_datastore_service.h"
31 #include "gnunet_statistics_service.h"
32 #include "datastore.h"
34 #define LOG(kind,...) GNUNET_log_from (kind, "datastore-api",__VA_ARGS__)
37 * Collect an instane number of statistics? May cause excessive IPC.
39 #define INSANE_STATISTICS GNUNET_NO
42 * If a client stopped asking for more results, how many more do
43 * we receive from the DB before killing the connection? Trade-off
44 * between re-doing TCP handshakes and (needlessly) receiving
47 #define MAX_EXCESS_RESULTS 8
50 * Context for processing status messages.
55 * Continuation to call with the status.
57 GNUNET_DATASTORE_ContinuationWithStatus cont;
60 * Closure for @e cont.
68 * Context for processing result messages.
73 * Function to call with the result.
75 GNUNET_DATASTORE_DatumProcessor proc;
78 * Closure for @e proc.
86 * Context for a queue operation.
91 struct StatusContext sc;
93 struct ResultContext rc;
99 * Entry in our priority queue.
101 struct GNUNET_DATASTORE_QueueEntry
105 * This is a linked list.
107 struct GNUNET_DATASTORE_QueueEntry *next;
110 * This is a linked list.
112 struct GNUNET_DATASTORE_QueueEntry *prev;
115 * Handle to the master context.
117 struct GNUNET_DATASTORE_Handle *h;
120 * Function to call after transmission of the request.
122 GNUNET_DATASTORE_ContinuationWithStatus cont;
125 * Closure for @e cont.
130 * Context for the operation.
132 union QueueContext qc;
135 * Envelope of the request to transmit, NULL after
138 struct GNUNET_MQ_Envelope *env;
141 * Priority in the queue.
143 unsigned int priority;
146 * Maximum allowed length of queue (otherwise
147 * this request should be discarded).
149 unsigned int max_queue;
152 * Expected response type.
154 uint16_t response_type;
160 * Handle to the datastore service.
162 struct GNUNET_DATASTORE_Handle
168 const struct GNUNET_CONFIGURATION_Handle *cfg;
171 * Current connection to the datastore service.
173 struct GNUNET_MQ_Handle *mq;
176 * Handle for statistics.
178 struct GNUNET_STATISTICS_Handle *stats;
181 * Current head of priority queue.
183 struct GNUNET_DATASTORE_QueueEntry *queue_head;
186 * Current tail of priority queue.
188 struct GNUNET_DATASTORE_QueueEntry *queue_tail;
191 * Task for trying to reconnect.
193 struct GNUNET_SCHEDULER_Task *reconnect_task;
196 * How quickly should we retry? Used for exponential back-off on
199 struct GNUNET_TIME_Relative retry_time;
202 * Number of entries in the queue.
204 unsigned int queue_size;
207 * Number of results we're receiving for the current query
208 * after application stopped to care. Used to determine when
209 * to reset the connection.
211 unsigned int result_count;
214 * We should ignore the next message(s) from the service.
216 unsigned int skip_next_messages;
222 * Try reconnecting to the datastore service.
224 * @param cls the `struct GNUNET_DATASTORE_Handle`
227 try_reconnect (void *cls);
231 * Disconnect from the service and then try reconnecting to the datastore service
234 * @param h handle to datastore to disconnect and reconnect
237 do_disconnect (struct GNUNET_DATASTORE_Handle *h)
244 GNUNET_MQ_destroy (h->mq);
246 h->skip_next_messages = 0;
248 = GNUNET_SCHEDULER_add_delayed (h->retry_time,
255 * Free a queue entry. Removes the given entry from the
256 * queue and releases associated resources. Does NOT
259 * @param qe entry to free.
262 free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
264 struct GNUNET_DATASTORE_Handle *h = qe->h;
266 GNUNET_CONTAINER_DLL_remove (h->queue_head,
275 * Handle error in sending drop request to datastore.
277 * @param cls closure with the datastore handle
278 * @param error error code
281 mq_error_handler (void *cls,
282 enum GNUNET_MQ_Error error)
284 struct GNUNET_DATASTORE_Handle *h = cls;
285 struct GNUNET_DATASTORE_QueueEntry *qe;
287 LOG (GNUNET_ERROR_TYPE_DEBUG,
288 "MQ error, reconnecting to DATASTORE\n");
294 union QueueContext qc = qe->qc;
295 uint16_t rt = qe->response_type;
297 LOG (GNUNET_ERROR_TYPE_DEBUG,
298 "Failed to receive response from database.\n");
299 free_queue_entry (qe);
302 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
303 if (NULL != qc.sc.cont)
304 qc.sc.cont (qc.sc.cont_cls,
306 GNUNET_TIME_UNIT_ZERO_ABS,
307 _("DATASTORE disconnected"));
309 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
310 if (NULL != qc.rc.proc)
311 qc.rc.proc (qc.rc.proc_cls,
315 GNUNET_TIME_UNIT_ZERO_ABS,
326 * Connect to the datastore service.
328 * @param cfg configuration to use
329 * @return handle to use to access the service
331 struct GNUNET_DATASTORE_Handle *
332 GNUNET_DATASTORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
334 struct GNUNET_DATASTORE_Handle *h;
336 LOG (GNUNET_ERROR_TYPE_DEBUG,
337 "Establishing DATASTORE connection!\n");
338 h = GNUNET_new (struct GNUNET_DATASTORE_Handle);
346 h->stats = GNUNET_STATISTICS_create ("datastore-api",
353 * Task used by to disconnect from the datastore after
354 * we send the #GNUNET_MESSAGE_TYPE_DATASTORE_DROP message.
356 * @param cls the datastore handle
359 disconnect_after_drop (void *cls)
361 struct GNUNET_DATASTORE_Handle *h = cls;
363 LOG (GNUNET_ERROR_TYPE_DEBUG,
364 "Drop sent, disconnecting\n");
365 GNUNET_DATASTORE_disconnect (h,
371 * Handle error in sending drop request to datastore.
373 * @param cls closure with the datastore handle
374 * @param error error code
377 disconnect_on_mq_error (void *cls,
378 enum GNUNET_MQ_Error error)
380 struct GNUNET_DATASTORE_Handle *h = cls;
382 LOG (GNUNET_ERROR_TYPE_ERROR,
383 "Failed to ask datastore to drop tables\n");
384 GNUNET_DATASTORE_disconnect (h,
390 * Disconnect from the datastore service (and free
391 * associated resources).
393 * @param h handle to the datastore
394 * @param drop set to #GNUNET_YES to delete all data in datastore (!)
397 GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
400 struct GNUNET_DATASTORE_QueueEntry *qe;
402 LOG (GNUNET_ERROR_TYPE_DEBUG,
403 "Datastore disconnect\n");
406 GNUNET_MQ_destroy (h->mq);
409 if (NULL != h->reconnect_task)
411 GNUNET_SCHEDULER_cancel (h->reconnect_task);
412 h->reconnect_task = NULL;
414 while (NULL != (qe = h->queue_head))
416 switch (qe->response_type)
418 case GNUNET_MESSAGE_TYPE_DATASTORE_STATUS:
419 if (NULL != qe->qc.sc.cont)
420 qe->qc.sc.cont (qe->qc.sc.cont_cls,
422 GNUNET_TIME_UNIT_ZERO_ABS,
423 _("Disconnected from DATASTORE"));
425 case GNUNET_MESSAGE_TYPE_DATASTORE_DATA:
426 if (NULL != qe->qc.rc.proc)
427 qe->qc.rc.proc (qe->qc.rc.proc_cls,
431 GNUNET_TIME_UNIT_ZERO_ABS,
437 free_queue_entry (qe);
439 if (GNUNET_YES == drop)
441 LOG (GNUNET_ERROR_TYPE_DEBUG,
442 "Re-connecting to issue DROP!\n");
443 GNUNET_assert (NULL == h->mq);
444 h->mq = GNUNET_CLIENT_connecT (h->cfg,
447 &disconnect_on_mq_error,
451 struct GNUNET_MessageHeader *hdr;
452 struct GNUNET_MQ_Envelope *env;
454 env = GNUNET_MQ_msg (hdr,
455 GNUNET_MESSAGE_TYPE_DATASTORE_DROP);
456 GNUNET_MQ_notify_sent (env,
457 &disconnect_after_drop,
459 GNUNET_MQ_send (h->mq,
465 GNUNET_STATISTICS_destroy (h->stats,
473 * Create a new entry for our priority queue (and possibly discard other entires if
474 * the queue is getting too long).
476 * @param h handle to the datastore
477 * @param env envelope with the message to queue
478 * @param queue_priority priority of the entry
479 * @param max_queue_size at what queue size should this request be dropped
480 * (if other requests of higher priority are in the queue)
481 * @param expected_type which type of response do we expect,
482 * #GNUNET_MESSAGE_TYPE_DATASTORE_STATUS or
483 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA
484 * @param qc client context (NOT a closure for @a response_proc)
485 * @return NULL if the queue is full
487 static struct GNUNET_DATASTORE_QueueEntry *
488 make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
489 struct GNUNET_MQ_Envelope *env,
490 unsigned int queue_priority,
491 unsigned int max_queue_size,
492 uint16_t expected_type,
493 const union QueueContext *qc)
495 struct GNUNET_DATASTORE_QueueEntry *qe;
496 struct GNUNET_DATASTORE_QueueEntry *pos;
501 while ( (NULL != pos) &&
502 (c < max_queue_size) &&
503 (pos->priority >= queue_priority) )
508 if (c >= max_queue_size)
510 GNUNET_STATISTICS_update (h->stats,
511 gettext_noop ("# queue overflows"),
514 GNUNET_MQ_discard (env);
517 qe = GNUNET_new (struct GNUNET_DATASTORE_QueueEntry);
520 qe->response_type = expected_type;
522 qe->priority = queue_priority;
523 qe->max_queue = max_queue_size;
526 /* append at the tail */
532 /* do not insert at HEAD if HEAD query was already
533 * transmitted and we are still receiving replies! */
534 if ( (NULL == pos) &&
535 (NULL == h->queue_head->env) )
539 #if INSANE_STATISTICS
540 GNUNET_STATISTICS_update (h->stats,
541 gettext_noop ("# queue entries created"),
545 GNUNET_CONTAINER_DLL_insert_after (h->queue_head,
555 * Process entries in the queue (or do nothing if we are already
558 * @param h handle to the datastore
561 process_queue (struct GNUNET_DATASTORE_Handle *h)
563 struct GNUNET_DATASTORE_QueueEntry *qe;
565 if (NULL == (qe = h->queue_head))
567 /* no entry in queue */
568 LOG (GNUNET_ERROR_TYPE_DEBUG,
574 /* waiting for replies */
575 LOG (GNUNET_ERROR_TYPE_DEBUG,
576 "Head request already transmitted\n");
581 /* waiting for reconnect */
582 LOG (GNUNET_ERROR_TYPE_DEBUG,
586 GNUNET_MQ_send (h->mq,
595 * Function called to check status message from the service.
598 * @param sm status message received
599 * @return #GNUNET_OK if the message is well-formed
602 check_status (void *cls,
603 const struct StatusMessage *sm)
605 uint16_t msize = ntohs (sm->header.size) - sizeof (*sm);
606 int32_t status = ntohl (sm->status);
610 const char *emsg = (const char *) &sm[1];
612 if ('\0' != emsg[msize - 1])
615 return GNUNET_SYSERR;
618 else if (GNUNET_SYSERR == status)
621 return GNUNET_SYSERR;
628 * Function called to handle status message from the service.
631 * @param sm status message received
634 handle_status (void *cls,
635 const struct StatusMessage *sm)
637 struct GNUNET_DATASTORE_Handle *h = cls;
638 struct GNUNET_DATASTORE_QueueEntry *qe;
639 struct StatusContext rc;
641 int32_t status = ntohl (sm->status);
643 if (h->skip_next_messages > 0)
645 h->skip_next_messages--;
649 if (NULL == (qe = h->queue_head))
661 if (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS != qe->response_type)
668 free_queue_entry (qe);
669 if (ntohs (sm->header.size) > sizeof (struct StatusMessage))
670 emsg = (const char *) &sm[1];
673 LOG (GNUNET_ERROR_TYPE_DEBUG,
674 "Received status %d/%s\n",
677 GNUNET_STATISTICS_update (h->stats,
678 gettext_noop ("# status messages received"),
681 h->retry_time = GNUNET_TIME_UNIT_ZERO;
684 rc.cont (rc.cont_cls,
686 GNUNET_TIME_absolute_ntoh (sm->min_expiration),
692 * Check data message we received from the service.
694 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
695 * @param dm message received
698 check_data (void *cls,
699 const struct DataMessage *dm)
701 uint16_t msize = ntohs (dm->header.size) - sizeof (*dm);
703 if (msize != ntohl (dm->size))
706 return GNUNET_SYSERR;
713 * Handle data message we got from the service.
715 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
716 * @param dm message received
719 handle_data (void *cls,
720 const struct DataMessage *dm)
722 struct GNUNET_DATASTORE_Handle *h = cls;
723 struct GNUNET_DATASTORE_QueueEntry *qe;
724 struct ResultContext rc;
726 if (h->skip_next_messages > 0)
744 if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type)
750 #if INSANE_STATISTICS
751 GNUNET_STATISTICS_update (h->stats,
752 gettext_noop ("# Results received"),
756 LOG (GNUNET_ERROR_TYPE_DEBUG,
757 "Received result %llu with type %u and size %u with key %s\n",
758 (unsigned long long) GNUNET_ntohll (dm->uid),
761 GNUNET_h2s (&dm->key));
763 free_queue_entry (qe);
764 h->retry_time = GNUNET_TIME_UNIT_ZERO;
767 rc.proc (rc.proc_cls,
772 ntohl (dm->priority),
773 ntohl (dm->anonymity),
774 GNUNET_TIME_absolute_ntoh (dm->expiration),
775 GNUNET_ntohll (dm->uid));
780 * Type of a function to call when we receive a
781 * #GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END message from the service.
783 * @param cls closure with the `struct GNUNET_DATASTORE_Handle *`
784 * @param msg message received
787 handle_data_end (void *cls,
788 const struct GNUNET_MessageHeader *msg)
790 struct GNUNET_DATASTORE_Handle *h = cls;
791 struct GNUNET_DATASTORE_QueueEntry *qe;
792 struct ResultContext rc;
794 if (h->skip_next_messages > 0)
796 h->skip_next_messages--;
813 if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type)
820 free_queue_entry (qe);
821 LOG (GNUNET_ERROR_TYPE_DEBUG,
822 "Received end of result set, new queue size is %u\n",
824 h->retry_time = GNUNET_TIME_UNIT_ZERO;
827 /* signal end of iteration */
829 rc.proc (rc.proc_cls,
836 GNUNET_TIME_UNIT_ZERO_ABS,
842 * Try reconnecting to the datastore service.
844 * @param cls the `struct GNUNET_DATASTORE_Handle`
847 try_reconnect (void *cls)
849 GNUNET_MQ_hd_var_size (status,
850 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
851 struct StatusMessage);
852 GNUNET_MQ_hd_var_size (data,
853 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
855 GNUNET_MQ_hd_fixed_size (data_end,
856 GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END,
857 struct GNUNET_MessageHeader);
858 struct GNUNET_DATASTORE_Handle *h = cls;
859 struct GNUNET_MQ_MessageHandler handlers[] = {
860 make_status_handler (h),
861 make_data_handler (h),
862 make_data_end_handler (h),
863 GNUNET_MQ_handler_end ()
866 h->retry_time = GNUNET_TIME_STD_BACKOFF (h->retry_time);
867 h->reconnect_task = NULL;
868 GNUNET_assert (NULL == h->mq);
869 h->mq = GNUNET_CLIENT_connecT (h->cfg,
876 GNUNET_STATISTICS_update (h->stats,
877 gettext_noop ("# datastore connections (re)created"),
880 LOG (GNUNET_ERROR_TYPE_DEBUG,
881 "Reconnected to DATASTORE\n");
887 * Dummy continuation used to do nothing (but be non-zero).
890 * @param result result
891 * @param min_expiration expiration time
892 * @param emsg error message
895 drop_status_cont (void *cls,
897 struct GNUNET_TIME_Absolute min_expiration,
905 * Store an item in the datastore. If the item is already present,
906 * the priorities are summed up and the higher expiration time and
907 * lower anonymity level is used.
909 * @param h handle to the datastore
910 * @param rid reservation ID to use (from "reserve"); use 0 if no
911 * prior reservation was made
912 * @param key key for the value
913 * @param size number of bytes in data
914 * @param data content stored
915 * @param type type of the content
916 * @param priority priority of the content
917 * @param anonymity anonymity-level for the content
918 * @param replication how often should the content be replicated to other peers?
919 * @param expiration expiration time for the content
920 * @param queue_priority ranking of this request in the priority queue
921 * @param max_queue_size at what queue size should this request be dropped
922 * (if other requests of higher priority are in the queue)
923 * @param cont continuation to call when done
924 * @param cont_cls closure for @a cont
925 * @return NULL if the entry was not queued, otherwise a handle that can be used to
926 * cancel; note that even if NULL is returned, the callback will be invoked
927 * (or rather, will already have been invoked)
929 struct GNUNET_DATASTORE_QueueEntry *
930 GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
932 const struct GNUNET_HashCode *key,
935 enum GNUNET_BLOCK_Type type,
938 uint32_t replication,
939 struct GNUNET_TIME_Absolute expiration,
940 unsigned int queue_priority,
941 unsigned int max_queue_size,
942 GNUNET_DATASTORE_ContinuationWithStatus cont,
945 struct GNUNET_DATASTORE_QueueEntry *qe;
946 struct GNUNET_MQ_Envelope *env;
947 struct DataMessage *dm;
948 union QueueContext qc;
950 if (size + sizeof (*dm) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
956 LOG (GNUNET_ERROR_TYPE_DEBUG,
957 "Asked to put %u bytes of data under key `%s' for %s\n",
960 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (expiration),
962 env = GNUNET_MQ_msg_extra (dm,
964 GNUNET_MESSAGE_TYPE_DATASTORE_PUT);
965 dm->rid = htonl (rid);
966 dm->size = htonl ((uint32_t) size);
967 dm->type = htonl (type);
968 dm->priority = htonl (priority);
969 dm->anonymity = htonl (anonymity);
970 dm->replication = htonl (replication);
971 dm->reserved = htonl (0);
972 dm->uid = GNUNET_htonll (0);
973 dm->expiration = GNUNET_TIME_absolute_hton (expiration);
979 qc.sc.cont_cls = cont_cls;
980 qe = make_queue_entry (h,
984 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
988 LOG (GNUNET_ERROR_TYPE_DEBUG,
989 "Could not create queue entry for PUT\n");
992 GNUNET_STATISTICS_update (h->stats,
993 gettext_noop ("# PUT requests executed"),
1002 * Reserve space in the datastore. This function should be used
1003 * to avoid "out of space" failures during a longer sequence of "put"
1004 * operations (for example, when a file is being inserted).
1006 * @param h handle to the datastore
1007 * @param amount how much space (in bytes) should be reserved (for content only)
1008 * @param entries how many entries will be created (to calculate per-entry overhead)
1009 * @param cont continuation to call when done; "success" will be set to
1010 * a positive reservation value if space could be reserved.
1011 * @param cont_cls closure for @a cont
1012 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1013 * cancel; note that even if NULL is returned, the callback will be invoked
1014 * (or rather, will already have been invoked)
1016 struct GNUNET_DATASTORE_QueueEntry *
1017 GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h,
1020 GNUNET_DATASTORE_ContinuationWithStatus cont,
1023 struct GNUNET_DATASTORE_QueueEntry *qe;
1024 struct GNUNET_MQ_Envelope *env;
1025 struct ReserveMessage *rm;
1026 union QueueContext qc;
1029 cont = &drop_status_cont;
1030 LOG (GNUNET_ERROR_TYPE_DEBUG,
1031 "Asked to reserve %llu bytes of data and %u entries\n",
1032 (unsigned long long) amount,
1033 (unsigned int) entries);
1034 env = GNUNET_MQ_msg (rm,
1035 GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE);
1036 rm->entries = htonl (entries);
1037 rm->amount = GNUNET_htonll (amount);
1040 qc.sc.cont_cls = cont_cls;
1041 qe = make_queue_entry (h,
1045 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1049 LOG (GNUNET_ERROR_TYPE_DEBUG,
1050 "Could not create queue entry to reserve\n");
1053 GNUNET_STATISTICS_update (h->stats,
1054 gettext_noop ("# RESERVE requests executed"),
1063 * Signal that all of the data for which a reservation was made has
1064 * been stored and that whatever excess space might have been reserved
1065 * can now be released.
1067 * @param h handle to the datastore
1068 * @param rid reservation ID (value of "success" in original continuation
1069 * from the "reserve" function).
1070 * @param queue_priority ranking of this request in the priority queue
1071 * @param max_queue_size at what queue size should this request be dropped
1072 * (if other requests of higher priority are in the queue)
1073 * @param queue_priority ranking of this request in the priority queue
1074 * @param max_queue_size at what queue size should this request be dropped
1075 * (if other requests of higher priority are in the queue)
1076 * @param cont continuation to call when done
1077 * @param cont_cls closure for @a cont
1078 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1079 * cancel; note that even if NULL is returned, the callback will be invoked
1080 * (or rather, will already have been invoked)
1082 struct GNUNET_DATASTORE_QueueEntry *
1083 GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
1085 unsigned int queue_priority,
1086 unsigned int max_queue_size,
1087 GNUNET_DATASTORE_ContinuationWithStatus cont,
1090 struct GNUNET_DATASTORE_QueueEntry *qe;
1091 struct GNUNET_MQ_Envelope *env;
1092 struct ReleaseReserveMessage *rrm;
1093 union QueueContext qc;
1096 cont = &drop_status_cont;
1097 LOG (GNUNET_ERROR_TYPE_DEBUG,
1098 "Asked to release reserve %d\n",
1100 env = GNUNET_MQ_msg (rrm,
1101 GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE);
1102 rrm->rid = htonl (rid);
1104 qc.sc.cont_cls = cont_cls;
1105 qe = make_queue_entry (h,
1109 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1113 LOG (GNUNET_ERROR_TYPE_DEBUG,
1114 "Could not create queue entry to release reserve\n");
1117 GNUNET_STATISTICS_update (h->stats,
1119 ("# RELEASE RESERVE requests executed"), 1,
1127 * Update a value in the datastore.
1129 * @param h handle to the datastore
1130 * @param uid identifier for the value
1131 * @param priority how much to increase the priority of the value
1132 * @param expiration new expiration value should be MAX of existing and this argument
1133 * @param queue_priority ranking of this request in the priority queue
1134 * @param max_queue_size at what queue size should this request be dropped
1135 * (if other requests of higher priority are in the queue)
1136 * @param cont continuation to call when done
1137 * @param cont_cls closure for @a cont
1138 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1139 * cancel; note that even if NULL is returned, the callback will be invoked
1140 * (or rather, will already have been invoked)
1142 struct GNUNET_DATASTORE_QueueEntry *
1143 GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
1146 struct GNUNET_TIME_Absolute expiration,
1147 unsigned int queue_priority,
1148 unsigned int max_queue_size,
1149 GNUNET_DATASTORE_ContinuationWithStatus cont,
1152 struct GNUNET_DATASTORE_QueueEntry *qe;
1153 struct GNUNET_MQ_Envelope *env;
1154 struct UpdateMessage *um;
1155 union QueueContext qc;
1158 cont = &drop_status_cont;
1159 LOG (GNUNET_ERROR_TYPE_DEBUG,
1160 "Asked to update entry %llu raising priority by %u and expiration to %s\n",
1162 (unsigned int) priority,
1163 GNUNET_STRINGS_absolute_time_to_string (expiration));
1164 env = GNUNET_MQ_msg (um,
1165 GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE);
1166 um->priority = htonl (priority);
1167 um->expiration = GNUNET_TIME_absolute_hton (expiration);
1168 um->uid = GNUNET_htonll (uid);
1171 qc.sc.cont_cls = cont_cls;
1172 qe = make_queue_entry (h,
1176 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1180 LOG (GNUNET_ERROR_TYPE_DEBUG,
1181 "Could not create queue entry for UPDATE\n");
1184 GNUNET_STATISTICS_update (h->stats,
1185 gettext_noop ("# UPDATE requests executed"), 1,
1193 * Explicitly remove some content from the database.
1194 * The @a cont continuation will be called with `status`
1195 * #GNUNET_OK" if content was removed, #GNUNET_NO
1196 * if no matching entry was found and #GNUNET_SYSERR
1197 * on all other types of errors.
1199 * @param h handle to the datastore
1200 * @param key key for the value
1201 * @param size number of bytes in data
1202 * @param data content stored
1203 * @param queue_priority ranking of this request in the priority queue
1204 * @param max_queue_size at what queue size should this request be dropped
1205 * (if other requests of higher priority are in the queue)
1206 * @param cont continuation to call when done
1207 * @param cont_cls closure for @a cont
1208 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1209 * cancel; note that even if NULL is returned, the callback will be invoked
1210 * (or rather, will already have been invoked)
1212 struct GNUNET_DATASTORE_QueueEntry *
1213 GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
1214 const struct GNUNET_HashCode *key,
1217 unsigned int queue_priority,
1218 unsigned int max_queue_size,
1219 GNUNET_DATASTORE_ContinuationWithStatus cont,
1222 struct GNUNET_DATASTORE_QueueEntry *qe;
1223 struct DataMessage *dm;
1224 struct GNUNET_MQ_Envelope *env;
1225 union QueueContext qc;
1227 if (sizeof (*dm) + size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
1233 cont = &drop_status_cont;
1234 LOG (GNUNET_ERROR_TYPE_DEBUG,
1235 "Asked to remove %u bytes under key `%s'\n",
1238 env = GNUNET_MQ_msg_extra (dm,
1240 GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
1241 dm->rid = htonl (0);
1242 dm->size = htonl (size);
1243 dm->type = htonl (0);
1244 dm->priority = htonl (0);
1245 dm->anonymity = htonl (0);
1246 dm->uid = GNUNET_htonll (0);
1247 dm->expiration = GNUNET_TIME_absolute_hton (GNUNET_TIME_UNIT_ZERO_ABS);
1254 qc.sc.cont_cls = cont_cls;
1256 qe = make_queue_entry (h,
1260 GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
1264 LOG (GNUNET_ERROR_TYPE_DEBUG,
1265 "Could not create queue entry for REMOVE\n");
1268 GNUNET_STATISTICS_update (h->stats,
1269 gettext_noop ("# REMOVE requests executed"),
1279 * Get a random value from the datastore for content replication.
1280 * Returns a single, random value among those with the highest
1281 * replication score, lowering positive replication scores by one for
1282 * the chosen value (if only content with a replication score exists,
1283 * a random value is returned and replication scores are not changed).
1285 * @param h handle to the datastore
1286 * @param queue_priority ranking of this request in the priority queue
1287 * @param max_queue_size at what queue size should this request be dropped
1288 * (if other requests of higher priority are in the queue)
1289 * @param proc function to call on a random value; it
1290 * will be called once with a value (if available)
1291 * and always once with a value of NULL.
1292 * @param proc_cls closure for @a proc
1293 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1296 struct GNUNET_DATASTORE_QueueEntry *
1297 GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
1298 unsigned int queue_priority,
1299 unsigned int max_queue_size,
1300 GNUNET_DATASTORE_DatumProcessor proc,
1303 struct GNUNET_DATASTORE_QueueEntry *qe;
1304 struct GNUNET_MQ_Envelope *env;
1305 struct GNUNET_MessageHeader *m;
1306 union QueueContext qc;
1308 GNUNET_assert (NULL != proc);
1309 LOG (GNUNET_ERROR_TYPE_DEBUG,
1310 "Asked to get replication entry\n");
1311 env = GNUNET_MQ_msg (m,
1312 GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION);
1314 qc.rc.proc_cls = proc_cls;
1315 qe = make_queue_entry (h,
1319 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1323 LOG (GNUNET_ERROR_TYPE_DEBUG,
1324 "Could not create queue entry for GET REPLICATION\n");
1327 GNUNET_STATISTICS_update (h->stats,
1329 ("# GET REPLICATION requests executed"), 1,
1337 * Get a single zero-anonymity value from the datastore.
1339 * @param h handle to the datastore
1340 * @param offset offset of the result (modulo num-results); set to
1341 * a random 64-bit value initially; then increment by
1342 * one each time; detect that all results have been found by uid
1343 * being again the first uid ever returned.
1344 * @param queue_priority ranking of this request in the priority queue
1345 * @param max_queue_size at what queue size should this request be dropped
1346 * (if other requests of higher priority are in the queue)
1347 * @param type allowed type for the operation (never zero)
1348 * @param proc function to call on a random value; it
1349 * will be called once with a value (if available)
1350 * or with NULL if none value exists.
1351 * @param proc_cls closure for @a proc
1352 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1355 struct GNUNET_DATASTORE_QueueEntry *
1356 GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
1358 unsigned int queue_priority,
1359 unsigned int max_queue_size,
1360 enum GNUNET_BLOCK_Type type,
1361 GNUNET_DATASTORE_DatumProcessor proc,
1364 struct GNUNET_DATASTORE_QueueEntry *qe;
1365 struct GNUNET_MQ_Envelope *env;
1366 struct GetZeroAnonymityMessage *m;
1367 union QueueContext qc;
1369 GNUNET_assert (NULL != proc);
1370 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
1371 LOG (GNUNET_ERROR_TYPE_DEBUG,
1372 "Asked to get %llu-th zero-anonymity entry of type %d\n",
1373 (unsigned long long) offset,
1375 env = GNUNET_MQ_msg (m,
1376 GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
1377 m->type = htonl ((uint32_t) type);
1378 m->offset = GNUNET_htonll (offset);
1380 qc.rc.proc_cls = proc_cls;
1381 qe = make_queue_entry (h,
1385 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1389 LOG (GNUNET_ERROR_TYPE_DEBUG,
1390 "Could not create queue entry for zero-anonymity procation\n");
1393 GNUNET_STATISTICS_update (h->stats,
1395 ("# GET ZERO ANONYMITY requests executed"), 1,
1403 * Get a result for a particular key from the datastore. The processor
1404 * will only be called once.
1406 * @param h handle to the datastore
1407 * @param offset offset of the result (modulo num-results); set to
1408 * a random 64-bit value initially; then increment by
1409 * one each time; detect that all results have been found by uid
1410 * being again the first uid ever returned.
1411 * @param key maybe NULL (to match all entries)
1412 * @param type desired type, 0 for any
1413 * @param queue_priority ranking of this request in the priority queue
1414 * @param max_queue_size at what queue size should this request be dropped
1415 * (if other requests of higher priority are in the queue)
1416 * @param proc function to call on each matching value;
1417 * will be called once with a NULL value at the end
1418 * @param proc_cls closure for @a proc
1419 * @return NULL if the entry was not queued, otherwise a handle that can be used to
1422 struct GNUNET_DATASTORE_QueueEntry *
1423 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
1425 const struct GNUNET_HashCode *key,
1426 enum GNUNET_BLOCK_Type type,
1427 unsigned int queue_priority,
1428 unsigned int max_queue_size,
1429 GNUNET_DATASTORE_DatumProcessor proc,
1432 struct GNUNET_DATASTORE_QueueEntry *qe;
1433 struct GNUNET_MQ_Envelope *env;
1434 struct GetKeyMessage *gkm;
1435 struct GetMessage *gm;
1436 union QueueContext qc;
1438 GNUNET_assert (NULL != proc);
1439 LOG (GNUNET_ERROR_TYPE_DEBUG,
1440 "Asked to look for data of type %u under key `%s'\n",
1441 (unsigned int) type,
1445 env = GNUNET_MQ_msg (gm,
1446 GNUNET_MESSAGE_TYPE_DATASTORE_GET);
1447 gm->type = htonl (type);
1448 gm->offset = GNUNET_htonll (offset);
1452 env = GNUNET_MQ_msg (gkm,
1453 GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
1454 gkm->type = htonl (type);
1455 gkm->offset = GNUNET_htonll (offset);
1459 qc.rc.proc_cls = proc_cls;
1460 qe = make_queue_entry (h,
1464 GNUNET_MESSAGE_TYPE_DATASTORE_DATA,
1468 LOG (GNUNET_ERROR_TYPE_DEBUG,
1469 "Could not queue request for `%s'\n",
1473 #if INSANE_STATISTICS
1474 GNUNET_STATISTICS_update (h->stats,
1475 gettext_noop ("# GET requests executed"),
1485 * Cancel a datastore operation. The final callback from the
1486 * operation must not have been done yet.
1488 * @param qe operation to cancel
1491 GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe)
1493 struct GNUNET_DATASTORE_Handle *h = qe->h;
1495 LOG (GNUNET_ERROR_TYPE_DEBUG,
1496 "Pending DATASTORE request %p cancelled (%d, %d)\n",
1499 h->queue_head == qe);
1500 if (NULL == qe->env)
1502 free_queue_entry (qe);
1503 h->skip_next_messages++;
1506 free_queue_entry (qe);
1511 /* end of datastore_api.c */