2 This file is part of GNUnet
3 (C) 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
28 #include "gnunet_statistics_service.h"
29 #include "plugin_datastore.h"
32 #define DEBUG_SQLITE GNUNET_YES
35 * After how many payload-changing operations
36 * do we sync our statistics?
38 #define MAX_STAT_SYNC_LAG 50
40 #define QUOTA_STAT_NAME gettext_noop ("file-sharing datastore utilization (in bytes)")
43 * Log an error message at log-level 'level' that indicates
44 * a failure of the command 'cmd' on file 'filename'
45 * with the message given by strerror(errno).
47 #define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed with error: %s\n"), cmd, sqlite3_errmsg(db->dbh)); } while(0)
49 #define SELECT_IT_LOW_PRIORITY_1 \
50 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio = ? AND hash > ?) "\
51 "ORDER BY hash ASC LIMIT 1"
53 #define SELECT_IT_LOW_PRIORITY_2 \
54 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio > ?) "\
55 "ORDER BY prio ASC, hash ASC LIMIT 1"
57 #define SELECT_IT_NON_ANONYMOUS_1 \
58 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio = ? AND hash < ? AND anonLevel = 0 AND expire > %llu) "\
59 " ORDER BY hash DESC LIMIT 1"
61 #define SELECT_IT_NON_ANONYMOUS_2 \
62 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (prio < ? AND anonLevel = 0 AND expire > %llu)"\
63 " ORDER BY prio DESC, hash DESC LIMIT 1"
65 #define SELECT_IT_EXPIRATION_TIME_1 \
66 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire = ? AND hash > ?) "\
67 " ORDER BY hash ASC LIMIT 1"
69 #define SELECT_IT_EXPIRATION_TIME_2 \
70 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire > ?) "\
71 " ORDER BY expire ASC, hash ASC LIMIT 1"
73 #define SELECT_IT_MIGRATION_ORDER_1 \
74 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire = ? AND hash < ?) "\
75 " ORDER BY hash DESC LIMIT 1"
77 #define SELECT_IT_MIGRATION_ORDER_2 \
78 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080 WHERE (expire < ? AND expire > %llu) "\
79 " ORDER BY expire DESC, hash DESC LIMIT 1"
82 * After how many ms "busy" should a DB operation fail for good?
83 * A low value makes sure that we are more responsive to requests
84 * (especially PUTs). A high value guarantees a higher success
85 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
87 * The default value of 250ms should ensure that users do not experience
88 * huge latencies while at the same time allowing operations to succeed
89 * with reasonable probability.
91 #define BUSY_TIMEOUT_MS 250
95 * Context for all functions in this plugin.
100 * Our execution environment.
102 struct GNUNET_DATASTORE_PluginEnvironment *env;
110 * Native SQLite database handle.
115 * Precompiled SQL for update.
117 sqlite3_stmt *updPrio;
120 * Precompiled SQL for insertion.
122 sqlite3_stmt *insertContent;
125 * Handle to the statistics service.
127 struct GNUNET_STATISTICS_Handle *statistics;
130 * How much data are we currently storing
133 unsigned long long payload;
136 * Number of updates that were made to the
137 * payload value since we last synchronized
138 * it with the statistics service.
140 unsigned int lastSync;
143 * Should the database be dropped on shutdown?
145 int drop_on_shutdown;
150 * @brief Prepare a SQL statement
152 * @param zSql SQL statement, UTF-8 encoded
155 sq_prepare (sqlite3 * dbh, const char *zSql,
156 sqlite3_stmt ** ppStmt)
159 return sqlite3_prepare (dbh,
161 strlen (zSql), ppStmt, (const char **) &dummy);
166 * Create our database indices.
169 create_indices (sqlite3 * dbh)
173 "CREATE INDEX idx_hash ON gn080 (hash)", NULL, NULL, NULL);
175 "CREATE INDEX idx_hash_vhash ON gn080 (hash,vhash)", NULL,
177 sqlite3_exec (dbh, "CREATE INDEX idx_prio ON gn080 (prio)", NULL, NULL,
179 sqlite3_exec (dbh, "CREATE INDEX idx_expire ON gn080 (expire)", NULL, NULL,
181 sqlite3_exec (dbh, "CREATE INDEX idx_comb3 ON gn080 (prio,anonLevel)", NULL,
183 sqlite3_exec (dbh, "CREATE INDEX idx_comb4 ON gn080 (prio,hash,anonLevel)",
185 sqlite3_exec (dbh, "CREATE INDEX idx_comb7 ON gn080 (expire,hash)", NULL,
192 #define CHECK(a) GNUNET_break(a)
196 #define ENULL_DEFINED 1
197 #define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERRROR, "%s\n", e); sqlite3_free(e); }
204 * Initialize the database connections and associated
205 * data structures (create tables and indices
206 * as needed as well).
208 * @return GNUNET_OK on success
211 database_setup (struct GNUNET_CONFIGURATION_Handle *cfg,
212 struct Plugin *plugin)
221 GNUNET_CONFIGURATION_get_value_filename (cfg,
226 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
228 _("Option `%s' in section `%s' missing in configuration!\n"),
231 return GNUNET_SYSERR;
233 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
236 GNUNET_free (afsdir);
237 return GNUNET_SYSERR;
239 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
241 nl_langinfo (CODESET)
243 "UTF-8" /* good luck */
246 GNUNET_free (afsdir);
248 /* Open database and precompile statements */
249 if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK)
251 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
253 _("Unable to initialize SQLite: %s.\n"),
254 sqlite3_errmsg (plugin->dbh));
255 return GNUNET_SYSERR;
258 sqlite3_exec (plugin->dbh,
259 "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
261 sqlite3_exec (plugin->dbh,
262 "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
264 sqlite3_exec (plugin->dbh,
265 "PRAGMA count_changes=OFF", NULL, NULL, ENULL));
267 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, ENULL));
269 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
272 /* We have to do it here, because otherwise precompiling SQL might fail */
274 sq_prepare (plugin->dbh,
275 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn080'",
277 if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
278 (sqlite3_exec (plugin->dbh,
279 "CREATE TABLE gn080 ("
280 " size INT4 NOT NULL DEFAULT 0,"
281 " type INT4 NOT NULL DEFAULT 0,"
282 " prio INT4 NOT NULL DEFAULT 0,"
283 " anonLevel INT4 NOT NULL DEFAULT 0,"
284 " expire INT8 NOT NULL DEFAULT 0,"
285 " hash TEXT NOT NULL DEFAULT '',"
286 " vhash TEXT NOT NULL DEFAULT '',"
287 " value BLOB NOT NULL DEFAULT '')", NULL, NULL,
288 NULL) != SQLITE_OK) )
290 LOG_SQLITE (plugin, NULL,
291 GNUNET_ERROR_TYPE_ERROR,
293 sqlite3_finalize (stmt);
294 return GNUNET_SYSERR;
296 sqlite3_finalize (stmt);
297 create_indices (plugin->dbh);
300 sq_prepare (plugin->dbh,
301 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn071'",
303 if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
304 (sqlite3_exec (plugin->dbh,
305 "CREATE TABLE gn071 ("
306 " key TEXT NOT NULL DEFAULT '',"
307 " value INTEGER NOT NULL DEFAULT 0)", NULL, NULL,
308 NULL) != SQLITE_OK) )
310 LOG_SQLITE (plugin, NULL,
311 GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
312 sqlite3_finalize (stmt);
313 return GNUNET_SYSERR;
315 sqlite3_finalize (stmt);
317 if ((sq_prepare (plugin->dbh,
318 "UPDATE gn080 SET prio = prio + ?, expire = MAX(expire,?) WHERE "
320 &plugin->updPrio) != SQLITE_OK) ||
321 (sq_prepare (plugin->dbh,
322 "INSERT INTO gn080 (size, type, prio, "
323 "anonLevel, expire, hash, vhash, value) VALUES "
324 "(?, ?, ?, ?, ?, ?, ?, ?)",
325 &plugin->insertContent) != SQLITE_OK))
327 LOG_SQLITE (plugin, NULL,
328 GNUNET_ERROR_TYPE_ERROR, "precompiling");
329 return GNUNET_SYSERR;
336 * Synchronize our utilization statistics with the
337 * statistics service.
340 sync_stats (struct Plugin *plugin)
342 GNUNET_STATISTICS_set (plugin->statistics,
346 plugin->lastSync = 0;
351 * Shutdown database connection and associate data
355 database_shutdown (struct Plugin *plugin)
357 if (plugin->lastSync > 0)
359 if (plugin->updPrio != NULL)
360 sqlite3_finalize (plugin->updPrio);
361 if (plugin->insertContent != NULL)
362 sqlite3_finalize (plugin->insertContent);
363 sqlite3_close (plugin->dbh);
364 GNUNET_free_non_null (plugin->fn);
369 * Get an estimate of how much space the database is
371 * @return number of bytes used on disk
373 static unsigned long long sqlite_plugin_get_size (void *cls)
375 struct Plugin *plugin = cls;
376 return plugin->payload;
381 * Delete the database entry with the given
385 delete_by_rowid (struct Plugin* plugin,
386 unsigned long long rid)
390 if (sq_prepare (plugin->dbh,
391 "DELETE FROM gn080 WHERE _ROWID_ = ?", &stmt) != SQLITE_OK)
393 LOG_SQLITE (plugin, NULL,
394 GNUNET_ERROR_TYPE_ERROR |
395 GNUNET_ERROR_TYPE_BULK, "sq_prepare");
396 return GNUNET_SYSERR;
398 sqlite3_bind_int64 (stmt, 1, rid);
399 if (SQLITE_DONE != sqlite3_step (stmt))
401 LOG_SQLITE (plugin, NULL,
402 GNUNET_ERROR_TYPE_ERROR |
403 GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
404 sqlite3_finalize (stmt);
405 return GNUNET_SYSERR;
407 sqlite3_finalize (stmt);
413 * Context for the universal iterator.
418 * Type of a function that will prepare
419 * the next iteration.
422 * @param nc the next context; NULL for the last
423 * call which gives the callback a chance to
424 * clean up the closure
425 * @return GNUNET_OK on success, GNUNET_NO if there are
426 * no more values, GNUNET_SYSERR on error
428 typedef int (*PrepareFunction)(void *cls,
429 struct NextContext *nc);
433 * Context we keep for the "next request" callback.
440 struct Plugin *plugin;
443 * Function to call on the next value.
453 * Function to call to prepare the next
456 PrepareFunction prep;
464 * Statement that the iterator will get the data
465 * from (updated or set by prep).
470 * Row ID of the last result.
472 unsigned long long last_rowid;
475 * Key of the last result.
477 GNUNET_HashCode lastKey;
480 * Expiration time of the last value visited.
482 struct GNUNET_TIME_Absolute lastExpiration;
485 * Priority of the last value visited.
487 unsigned int lastPriority;
490 * Number of results processed so far.
495 * Set to GNUNET_YES if we must stop now.
502 * Continuation of "sqlite_next_request".
504 * @param cls the next context
507 sqlite_next_request_cont (void *cls,
508 const struct GNUNET_SCHEDULER_TaskContext *tc)
510 static struct GNUNET_TIME_Absolute zero;
511 struct NextContext * nc= cls;
512 struct Plugin *plugin;
513 unsigned long long rowid;
518 unsigned int priority;
519 unsigned int anonymity;
520 struct GNUNET_TIME_Absolute expiration;
521 const GNUNET_HashCode *key;
526 if ( (GNUNET_YES == nc->end_it) ||
527 (GNUNET_OK != (nc->prep(nc->prep_cls,
531 nc->iter (nc->iter_cls,
532 NULL, NULL, 0, NULL, 0, 0, 0,
534 nc->prep (nc->prep_cls, NULL);
539 rowid = sqlite3_column_int64 (nc->stmt, 7);
540 nc->last_rowid = rowid;
541 type = sqlite3_column_int (nc->stmt, 1);
542 size = sqlite3_column_bytes (nc->stmt, 6);
543 if (sqlite3_column_bytes (nc->stmt, 5) != sizeof (GNUNET_HashCode))
545 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
547 _("Invalid data in database. Trying to fix (by deletion).\n"));
548 if (SQLITE_OK != sqlite3_reset (nc->stmt))
549 LOG_SQLITE (nc->plugin, NULL,
550 GNUNET_ERROR_TYPE_ERROR |
551 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
554 "DELETE FROM gn080 WHERE NOT LENGTH(hash) = ?",
555 &stmtd) != SQLITE_OK)
557 LOG_SQLITE (nc->plugin, NULL,
558 GNUNET_ERROR_TYPE_ERROR |
559 GNUNET_ERROR_TYPE_BULK,
564 if (SQLITE_OK != sqlite3_bind_int (stmtd, 1, sizeof (GNUNET_HashCode)))
565 LOG_SQLITE (nc->plugin, NULL,
566 GNUNET_ERROR_TYPE_ERROR |
567 GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_int");
568 if (SQLITE_DONE != sqlite3_step (stmtd))
569 LOG_SQLITE (nc->plugin, NULL,
570 GNUNET_ERROR_TYPE_ERROR |
571 GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
572 if (SQLITE_OK != sqlite3_finalize (stmtd))
573 LOG_SQLITE (nc->plugin, NULL,
574 GNUNET_ERROR_TYPE_ERROR |
575 GNUNET_ERROR_TYPE_BULK, "sqlite3_finalize");
579 priority = sqlite3_column_int (nc->stmt, 2);
580 anonymity = sqlite3_column_int (nc->stmt, 3);
581 expiration.value = sqlite3_column_int64 (nc->stmt, 4);
582 key = sqlite3_column_blob (nc->stmt, 5);
583 nc->lastPriority = priority;
584 nc->lastExpiration = expiration;
585 memcpy (&nc->lastKey, key, sizeof(GNUNET_HashCode));
586 data = sqlite3_column_blob (nc->stmt, 6);
588 ret = nc->iter (nc->iter_cls,
598 if (ret == GNUNET_SYSERR)
600 nc->end_it = GNUNET_YES;
603 if ( (ret == GNUNET_NO) &&
604 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
606 plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
608 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
615 * Function invoked on behalf of a "PluginIterator"
616 * asking the database plugin to call the iterator
617 * with the next item.
619 * @param next_cls whatever argument was given
620 * to the PluginIterator as "next_cls".
621 * @param end_it set to GNUNET_YES if we
622 * should terminate the iteration early
623 * (iterator should be still called once more
624 * to signal the end of the iteration).
627 sqlite_next_request (void *next_cls,
630 struct NextContext * nc= next_cls;
632 if (GNUNET_YES == end_it)
633 nc->end_it = GNUNET_YES;
634 GNUNET_SCHEDULER_add_continuation (nc->plugin->env->sched,
636 &sqlite_next_request_cont,
638 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
644 * Store an item in the datastore.
647 * @param key key for the item
648 * @param size number of bytes in data
649 * @param data content stored
650 * @param type type of the content
651 * @param priority priority of the content
652 * @param anonymity anonymity-level for the content
653 * @param expiration expiration time for the content
654 * @param msg set to an error message
655 * @return GNUNET_OK on success
658 sqlite_plugin_put (void *cls,
659 const GNUNET_HashCode * key,
665 struct GNUNET_TIME_Absolute expiration,
668 struct Plugin *plugin = cls;
671 GNUNET_HashCode vhash;
674 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
676 "Storing in database block with type %u/key `%s'/priority %u/expiration %llu (%lld).\n",
680 (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).value,
681 (long long) expiration.value);
683 GNUNET_CRYPTO_hash (data, size, &vhash);
684 stmt = plugin->insertContent;
685 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, size)) ||
686 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
687 (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
688 (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
689 (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, (sqlite3_int64) expiration.value)) ||
691 sqlite3_bind_blob (stmt, 6, key, sizeof (GNUNET_HashCode),
692 SQLITE_TRANSIENT)) ||
694 sqlite3_bind_blob (stmt, 7, &vhash, sizeof (GNUNET_HashCode),
697 sqlite3_bind_blob (stmt, 8, data, size,
702 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX");
703 if (SQLITE_OK != sqlite3_reset (stmt))
704 LOG_SQLITE (plugin, NULL,
705 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
706 return GNUNET_SYSERR;
708 n = sqlite3_step (stmt);
709 if (n != SQLITE_DONE)
711 if (n == SQLITE_BUSY)
713 LOG_SQLITE (plugin, msg,
714 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
715 sqlite3_reset (stmt);
719 LOG_SQLITE (plugin, msg,
720 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
721 sqlite3_reset (stmt);
722 return GNUNET_SYSERR;
724 if (SQLITE_OK != sqlite3_reset (stmt))
725 LOG_SQLITE (plugin, NULL,
726 GNUNET_ERROR_TYPE_ERROR |
727 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
729 plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
730 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
737 * Update the priority for a particular key in the datastore. If
738 * the expiration time in value is different than the time found in
739 * the datastore, the higher value should be kept. For the
740 * anonymity level, the lower value is to be used. The specified
741 * priority should be added to the existing priority, ignoring the
744 * Note that it is possible for multiple values to match this put.
745 * In that case, all of the respective values are updated.
747 * @param uid unique identifier of the datum
748 * @param delta by how much should the priority
749 * change? If priority + delta < 0 the
750 * priority should be set to 0 (never go
752 * @param expire new expiration time should be the
753 * MAX of any existing expiration time and
755 * @param msg set to an error message
756 * @return GNUNET_OK on success
759 sqlite_plugin_update (void *cls,
761 int delta, struct GNUNET_TIME_Absolute expire,
764 struct Plugin *plugin = cls;
767 sqlite3_bind_int (plugin->updPrio, 1, delta);
768 sqlite3_bind_int64 (plugin->updPrio, 2, expire.value);
769 sqlite3_bind_int64 (plugin->updPrio, 3, uid);
770 n = sqlite3_step (plugin->updPrio);
772 LOG_SQLITE (plugin, msg,
773 GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
777 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
781 sqlite3_reset (plugin->updPrio);
783 if (n == SQLITE_BUSY)
785 return n == SQLITE_OK ? GNUNET_OK : GNUNET_SYSERR;
791 sqlite3_stmt *stmt_1;
792 sqlite3_stmt *stmt_2;
796 int limit_nonanonymous;
802 iter_next_prepare (void *cls,
803 struct NextContext *nc)
805 struct IterContext *ic = cls;
806 struct Plugin *plugin;
812 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
813 "Asked to clean up iterator state.\n");
815 sqlite3_finalize (ic->stmt_1);
816 sqlite3_finalize (ic->stmt_2);
817 return GNUNET_SYSERR;
819 sqlite3_reset (ic->stmt_1);
820 sqlite3_reset (ic->stmt_2);
825 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
826 "Restricting to results larger than the last priority %u\n",
829 sqlite3_bind_int (ic->stmt_1, 1, nc->lastPriority);
830 sqlite3_bind_int (ic->stmt_2, 1, nc->lastPriority);
835 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
836 "Restricting to results larger than the last expiration %llu\n",
837 (unsigned long long) nc->lastExpiration.value);
839 sqlite3_bind_int64 (ic->stmt_1, 1, nc->lastExpiration.value);
840 sqlite3_bind_int64 (ic->stmt_2, 1, nc->lastExpiration.value);
843 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
844 "Restricting to results larger than the last key `%s'\n",
845 GNUNET_h2s(&nc->lastKey));
847 sqlite3_bind_blob (ic->stmt_1, 2,
849 sizeof (GNUNET_HashCode),
851 if (SQLITE_ROW == (ret = sqlite3_step (ic->stmt_1)))
854 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
855 "Result found using iterator 1\n");
857 nc->stmt = ic->stmt_1;
860 if (ret != SQLITE_DONE)
862 LOG_SQLITE (plugin, NULL,
863 GNUNET_ERROR_TYPE_ERROR |
864 GNUNET_ERROR_TYPE_BULK,
866 return GNUNET_SYSERR;
868 if (SQLITE_OK != sqlite3_reset (ic->stmt_1))
869 LOG_SQLITE (plugin, NULL,
870 GNUNET_ERROR_TYPE_ERROR |
871 GNUNET_ERROR_TYPE_BULK,
873 if (SQLITE_ROW == (ret = sqlite3_step (ic->stmt_2)))
876 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
877 "Result found using iterator 2\n");
879 nc->stmt = ic->stmt_2;
882 if (ret != SQLITE_DONE)
884 LOG_SQLITE (plugin, NULL,
885 GNUNET_ERROR_TYPE_ERROR |
886 GNUNET_ERROR_TYPE_BULK,
888 return GNUNET_SYSERR;
890 if (SQLITE_OK != sqlite3_reset (ic->stmt_2))
891 LOG_SQLITE (plugin, NULL,
892 GNUNET_ERROR_TYPE_ERROR |
893 GNUNET_ERROR_TYPE_BULK,
896 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
897 "No result found using either iterator\n");
904 * Call a method for each key in the database and
905 * call the callback method on it.
907 * @param type entries of which type should be considered?
908 * @param iter function to call on each matching value;
909 * will be called once with a NULL value at the end
910 * @param iter_cls closure for iter
911 * @return the number of results processed,
912 * GNUNET_SYSERR on error
915 basic_iter (struct Plugin *plugin,
920 int limit_nonanonymous,
921 const char *stmt_str_1,
922 const char *stmt_str_2,
926 static struct GNUNET_TIME_Absolute zero;
927 struct NextContext *nc;
928 struct IterContext *ic;
929 sqlite3_stmt *stmt_1;
930 sqlite3_stmt *stmt_2;
933 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
934 "At %llu, using queries `%s' and `%s'\n",
935 (unsigned long long) GNUNET_TIME_absolute_get ().value,
939 if (sq_prepare (plugin->dbh, stmt_str_1, &stmt_1) != SQLITE_OK)
941 LOG_SQLITE (plugin, NULL,
942 GNUNET_ERROR_TYPE_ERROR |
943 GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare");
944 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
947 if (sq_prepare (plugin->dbh, stmt_str_2, &stmt_2) != SQLITE_OK)
949 LOG_SQLITE (plugin, NULL,
950 GNUNET_ERROR_TYPE_ERROR |
951 GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare");
952 sqlite3_finalize (stmt_1);
953 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
956 nc = GNUNET_malloc (sizeof(struct NextContext) +
957 sizeof(struct IterContext));
960 nc->iter_cls = iter_cls;
962 ic = (struct IterContext*) &nc[1];
967 ic->is_prio = is_prio;
968 ic->is_migr = is_migr;
969 ic->limit_nonanonymous = limit_nonanonymous;
970 nc->prep = &iter_next_prepare;
974 nc->lastPriority = 0;
975 nc->lastExpiration.value = 0;
976 memset (&nc->lastKey, 0, sizeof (GNUNET_HashCode));
980 nc->lastPriority = 0x7FFFFFFF;
981 nc->lastExpiration.value = 0x7FFFFFFFFFFFFFFFLL;
982 memset (&nc->lastKey, 255, sizeof (GNUNET_HashCode));
984 sqlite_next_request (nc, GNUNET_NO);
989 * Select a subset of the items in the datastore and call
990 * the given iterator for each of them.
992 * @param type entries of which type should be considered?
993 * Use 0 for any type.
994 * @param iter function to call on each matching value;
995 * will be called once with a NULL value at the end
996 * @param iter_cls closure for iter
999 sqlite_plugin_iter_low_priority (void *cls,
1001 PluginIterator iter,
1006 GNUNET_YES, GNUNET_YES,
1007 GNUNET_NO, GNUNET_NO,
1008 SELECT_IT_LOW_PRIORITY_1,
1009 SELECT_IT_LOW_PRIORITY_2,
1015 * Select a subset of the items in the datastore and call
1016 * the given iterator for each of them.
1018 * @param type entries of which type should be considered?
1019 * Use 0 for any type.
1020 * @param iter function to call on each matching value;
1021 * will be called once with a NULL value at the end
1022 * @param iter_cls closure for iter
1025 sqlite_plugin_iter_zero_anonymity (void *cls,
1027 PluginIterator iter,
1030 struct GNUNET_TIME_Absolute now;
1034 now = GNUNET_TIME_absolute_get ();
1035 GNUNET_asprintf (&q1, SELECT_IT_NON_ANONYMOUS_1,
1036 (unsigned long long) now.value);
1037 GNUNET_asprintf (&q2, SELECT_IT_NON_ANONYMOUS_2,
1038 (unsigned long long) now.value);
1041 GNUNET_NO, GNUNET_YES,
1042 GNUNET_NO, GNUNET_YES,
1053 * Select a subset of the items in the datastore and call
1054 * the given iterator for each of them.
1056 * @param type entries of which type should be considered?
1057 * Use 0 for any type.
1058 * @param iter function to call on each matching value;
1059 * will be called once with a NULL value at the end
1060 * @param iter_cls closure for iter
1063 sqlite_plugin_iter_ascending_expiration (void *cls,
1065 PluginIterator iter,
1068 struct GNUNET_TIME_Absolute now;
1072 now = GNUNET_TIME_absolute_get ();
1073 GNUNET_asprintf (&q1, SELECT_IT_EXPIRATION_TIME_1,
1074 (unsigned long long) 0*now.value);
1075 GNUNET_asprintf (&q2, SELECT_IT_EXPIRATION_TIME_2,
1076 (unsigned long long) 0*now.value);
1079 GNUNET_YES, GNUNET_NO,
1080 GNUNET_NO, GNUNET_NO,
1089 * Select a subset of the items in the datastore and call
1090 * the given iterator for each of them.
1092 * @param type entries of which type should be considered?
1093 * Use 0 for any type.
1094 * @param iter function to call on each matching value;
1095 * will be called once with a NULL value at the end
1096 * @param iter_cls closure for iter
1099 sqlite_plugin_iter_migration_order (void *cls,
1101 PluginIterator iter,
1104 struct GNUNET_TIME_Absolute now;
1107 now = GNUNET_TIME_absolute_get ();
1108 GNUNET_asprintf (&q, SELECT_IT_MIGRATION_ORDER_2,
1109 (unsigned long long) now.value);
1112 GNUNET_NO, GNUNET_NO,
1113 GNUNET_YES, GNUNET_NO,
1114 SELECT_IT_MIGRATION_ORDER_1,
1122 all_next_prepare (void *cls,
1123 struct NextContext *nc)
1125 struct Plugin *plugin;
1131 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1132 "Asked to clean up iterator state.\n");
1134 return GNUNET_SYSERR;
1136 plugin = nc->plugin;
1137 if (SQLITE_ROW == (ret = sqlite3_step (nc->stmt)))
1141 if (ret != SQLITE_DONE)
1143 LOG_SQLITE (plugin, NULL,
1144 GNUNET_ERROR_TYPE_ERROR |
1145 GNUNET_ERROR_TYPE_BULK,
1147 return GNUNET_SYSERR;
1154 * Select a subset of the items in the datastore and call
1155 * the given iterator for each of them.
1157 * @param type entries of which type should be considered?
1158 * Use 0 for any type.
1159 * @param iter function to call on each matching value;
1160 * will be called once with a NULL value at the end
1161 * @param iter_cls closure for iter
1164 sqlite_plugin_iter_all_now (void *cls,
1166 PluginIterator iter,
1169 static struct GNUNET_TIME_Absolute zero;
1170 struct Plugin *plugin = cls;
1171 struct NextContext *nc;
1174 if (sq_prepare (plugin->dbh,
1175 "SELECT size,type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn080",
1176 &stmt) != SQLITE_OK)
1178 LOG_SQLITE (plugin, NULL,
1179 GNUNET_ERROR_TYPE_ERROR |
1180 GNUNET_ERROR_TYPE_BULK, "sqlite3_prepare");
1181 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1184 nc = GNUNET_malloc (sizeof(struct NextContext));
1185 nc->plugin = plugin;
1187 nc->iter_cls = iter_cls;
1189 nc->prep = &all_next_prepare;
1190 nc->prep_cls = NULL;
1191 sqlite_next_request (nc, GNUNET_NO);
1195 struct GetNextContext
1202 GNUNET_HashCode key;
1203 GNUNET_HashCode vhash;
1208 get_next_prepare (void *cls,
1209 struct NextContext *nc)
1211 struct GetNextContext *gnc = cls;
1218 sqlite3_finalize (gnc->stmt);
1219 return GNUNET_SYSERR;
1221 if (nc->count == gnc->total)
1223 if (nc->count + gnc->off == gnc->total)
1226 limit_off = gnc->off;
1230 sqlite3_reset (nc->stmt);
1231 ret = sqlite3_bind_blob (nc->stmt,
1234 sizeof (GNUNET_HashCode),
1236 if ((gnc->have_vhash) && (ret == SQLITE_OK))
1237 ret = sqlite3_bind_blob (nc->stmt,
1240 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
1241 if ((gnc->type != 0) && (ret == SQLITE_OK))
1242 ret = sqlite3_bind_int (nc->stmt, sqoff++, gnc->type);
1243 if (ret == SQLITE_OK)
1244 ret = sqlite3_bind_int64 (nc->stmt, sqoff++, nc->last_rowid + 1);
1245 if (ret == SQLITE_OK)
1246 ret = sqlite3_bind_int (nc->stmt, sqoff++, limit_off);
1247 if (ret != SQLITE_OK)
1248 return GNUNET_SYSERR;
1249 if (SQLITE_ROW != sqlite3_step (nc->stmt))
1256 * Iterate over the results for a particular key
1259 * @param cls closure
1260 * @param key maybe NULL (to match all entries)
1261 * @param vhash hash of the value, maybe NULL (to
1262 * match all values that have the right key).
1263 * Note that for DBlocks there is no difference
1264 * betwen key and vhash, but for other blocks
1266 * @param type entries of which type are relevant?
1267 * Use 0 for any type.
1268 * @param iter function to call on each matching value;
1269 * will be called once with a NULL value at the end
1270 * @param iter_cls closure for iter
1273 sqlite_plugin_get (void *cls,
1274 const GNUNET_HashCode * key,
1275 const GNUNET_HashCode * vhash,
1277 PluginIterator iter, void *iter_cls)
1279 static struct GNUNET_TIME_Absolute zero;
1280 struct Plugin *plugin = cls;
1281 struct GetNextContext *gpc;
1282 struct NextContext *nc;
1289 GNUNET_assert (iter != NULL);
1292 sqlite_plugin_iter_low_priority (cls, type, iter, iter_cls);
1295 GNUNET_snprintf (scratch, 256,
1296 "SELECT count(*) FROM gn080 WHERE hash=:1%s%s",
1297 vhash == NULL ? "" : " AND vhash=:2",
1298 type == 0 ? "" : (vhash ==
1299 NULL) ? " AND type=:2" : " AND type=:3");
1300 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
1302 LOG_SQLITE (plugin, NULL,
1303 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
1304 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1308 ret = sqlite3_bind_blob (stmt,
1310 key, sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
1311 if ((vhash != NULL) && (ret == SQLITE_OK))
1312 ret = sqlite3_bind_blob (stmt,
1315 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
1316 if ((type != 0) && (ret == SQLITE_OK))
1317 ret = sqlite3_bind_int (stmt, sqoff++, type);
1318 if (SQLITE_OK != ret)
1320 LOG_SQLITE (plugin, NULL,
1321 GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
1322 sqlite3_reset (stmt);
1323 sqlite3_finalize (stmt);
1324 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1327 ret = sqlite3_step (stmt);
1328 if (ret != SQLITE_ROW)
1330 LOG_SQLITE (plugin, NULL,
1331 GNUNET_ERROR_TYPE_ERROR| GNUNET_ERROR_TYPE_BULK,
1333 sqlite3_reset (stmt);
1334 sqlite3_finalize (stmt);
1335 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1338 total = sqlite3_column_int (stmt, 0);
1339 sqlite3_reset (stmt);
1340 sqlite3_finalize (stmt);
1343 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1347 GNUNET_snprintf (scratch, 256,
1348 "SELECT size, type, prio, anonLevel, expire, hash, value, _ROWID_ "
1349 "FROM gn080 WHERE hash=:1%s%s AND _ROWID_ >= :%d "
1350 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET :d",
1351 vhash == NULL ? "" : " AND vhash=:2",
1352 type == 0 ? "" : (vhash ==
1353 NULL) ? " AND type=:2" : " AND type=:3",
1355 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
1357 LOG_SQLITE (plugin, NULL,
1358 GNUNET_ERROR_TYPE_ERROR |
1359 GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
1360 iter (iter_cls, NULL, NULL, 0, NULL, 0, 0, 0, zero, 0);
1363 nc = GNUNET_malloc (sizeof(struct NextContext) +
1364 sizeof(struct GetNextContext));
1365 nc->plugin = plugin;
1367 nc->iter_cls = iter_cls;
1369 gpc = (struct GetNextContext*) &nc[1];
1373 gpc->stmt = stmt; /* alias used for freeing at the end! */
1376 gpc->have_vhash = GNUNET_YES;
1377 gpc->vhash = *vhash;
1379 gpc->off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
1380 nc->prep = &get_next_prepare;
1382 sqlite_next_request (nc, GNUNET_NO);
1390 sqlite_plugin_drop (void *cls)
1392 struct Plugin *plugin = cls;
1393 plugin->drop_on_shutdown = GNUNET_YES;
1398 * Callback function to process statistic values.
1400 * @param cls closure
1401 * @param subsystem name of subsystem that created the statistic
1402 * @param name the name of the datum
1403 * @param value the current value
1404 * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1405 * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1408 process_stat_in (void *cls,
1409 const char *subsystem,
1411 unsigned long long value,
1414 struct Plugin *plugin = cls;
1415 plugin->payload += value;
1421 * Entry point for the plugin.
1424 libgnunet_plugin_datastore_sqlite_init (void *cls)
1426 static struct Plugin plugin;
1427 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1428 struct GNUNET_DATASTORE_PluginFunctions *api;
1430 if (plugin.env != NULL)
1431 return NULL; /* can only initialize once! */
1432 memset (&plugin, 0, sizeof(struct Plugin));
1434 plugin.statistics = GNUNET_STATISTICS_create (env->sched,
1437 GNUNET_STATISTICS_get (plugin.statistics,
1440 GNUNET_TIME_UNIT_MINUTES,
1445 database_setup (env->cfg, &plugin))
1447 database_shutdown (&plugin);
1450 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1452 api->get_size = &sqlite_plugin_get_size;
1453 api->put = &sqlite_plugin_put;
1454 api->next_request = &sqlite_next_request;
1455 api->get = &sqlite_plugin_get;
1456 api->update = &sqlite_plugin_update;
1457 api->iter_low_priority = &sqlite_plugin_iter_low_priority;
1458 api->iter_zero_anonymity = &sqlite_plugin_iter_zero_anonymity;
1459 api->iter_ascending_expiration = &sqlite_plugin_iter_ascending_expiration;
1460 api->iter_migration_order = &sqlite_plugin_iter_migration_order;
1461 api->iter_all_now = &sqlite_plugin_iter_all_now;
1462 api->drop = &sqlite_plugin_drop;
1463 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1464 "sqlite", _("Sqlite database running\n"));
1470 * Exit point from the plugin.
1473 libgnunet_plugin_datastore_sqlite_done (void *cls)
1476 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1477 struct Plugin *plugin = api->cls;
1480 if (plugin->drop_on_shutdown)
1481 fn = GNUNET_strdup (plugin->fn);
1482 database_shutdown (plugin);
1484 plugin->payload = 0;
1488 if (0 != UNLINK(fn))
1489 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1497 /* end of plugin_datastore_sqlite.c */