2 * This file is part of GNUnet
3 * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
5 * GNUnet is free software: you can redistribute it and/or modify it
6 * under the terms of the GNU Affero General Public License as published
7 * by the Free Software Foundation, either version 3 of the License,
8 * or (at your option) any later version.
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Affero General Public License for more details.
15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
28 #include "gnunet_datastore_plugin.h"
29 #include "gnunet_sq_lib.h"
34 * We allocate items on the stack at times. To prevent a stack
35 * overflow, we impose a limit on the maximum size for the data per
36 * item. 64k should be enough.
38 #define MAX_ITEM_SIZE 65536
41 * After how many ms "busy" should a DB operation fail for good?
42 * A low value makes sure that we are more responsive to requests
43 * (especially PUTs). A high value guarantees a higher success
44 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
46 * The default value of 250ms should ensure that users do not experience
47 * huge latencies while at the same time allowing operations to succeed
48 * with reasonable probability.
50 #define BUSY_TIMEOUT_MS 250
54 * Log an error message at log-level 'level' that indicates
55 * a failure of the command 'cmd' on file 'filename'
56 * with the message given by strerror(errno).
58 #define LOG_SQLITE(db, level, cmd) \
61 GNUNET_log_from (level, \
63 _ ("`%s' failed at %s:%d with error: %s\n"), \
67 sqlite3_errmsg (db->dbh)); \
72 * Log an error message at log-level 'level' that indicates
73 * a failure of the command 'cmd' on file 'filename'
74 * with the message given by strerror(errno).
76 #define LOG_SQLITE_MSG(db, msg, level, cmd) \
79 GNUNET_log_from (level, \
81 _ ("`%s' failed at %s:%d with error: %s\n"), \
85 sqlite3_errmsg (db->dbh)); \
86 GNUNET_asprintf (msg, \
87 _ ("`%s' failed at %s:%u with error: %s"), \
91 sqlite3_errmsg (db->dbh)); \
96 * Context for all functions in this plugin.
101 * Our execution environment.
103 struct GNUNET_DATASTORE_PluginEnvironment *env;
111 * Native SQLite database handle.
116 * Precompiled SQL for remove_key.
118 sqlite3_stmt *remove;
121 * Precompiled SQL for deletion.
123 sqlite3_stmt *delRow;
126 * Precompiled SQL for update.
128 sqlite3_stmt *update;
131 * Get maximum repl value in database.
133 sqlite3_stmt *maxRepl;
136 * Precompiled SQL for replication decrement.
138 sqlite3_stmt *updRepl;
141 * Precompiled SQL for replication selection.
143 sqlite3_stmt *selRepl;
146 * Precompiled SQL for expiration selection.
148 sqlite3_stmt *selExpi;
151 * Precompiled SQL for expiration selection.
153 sqlite3_stmt *selZeroAnon;
156 * Precompiled SQL for insertion.
158 sqlite3_stmt *insertContent;
161 * Precompiled SQL for selection
163 sqlite3_stmt *get[8];
166 * Should the database be dropped on shutdown?
168 int drop_on_shutdown;
173 * @brief Prepare a SQL statement
175 * @param dbh handle to the database
176 * @param zSql SQL statement, UTF-8 encoded
177 * @param ppStmt set to the prepared statement
178 * @return 0 on success
181 sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
186 result = sqlite3_prepare_v2 (dbh,
190 (const char **) &dummy);
191 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
193 "Prepared `%s' / %p: %d\n",
202 * Create our database indices.
204 * @param dbh handle to the database
207 create_indices (sqlite3 *dbh)
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
238 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
246 #define CHECK(a) GNUNET_break (a)
250 #define ENULL_DEFINED 1
254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \
261 * Initialize the database connections and associated
262 * data structures (create tables and indices
263 * as needed as well).
265 * @param cfg our configuration
266 * @param plugin the plugin context (state for this module)
267 * @return #GNUNET_OK on success
270 database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
271 struct Plugin *plugin)
280 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
285 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
288 return GNUNET_SYSERR;
290 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
292 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
295 GNUNET_free (afsdir);
296 return GNUNET_SYSERR;
298 /* database is new or got deleted, reset payload to zero! */
299 if (NULL != plugin->env->duc)
300 plugin->env->duc (plugin->env->cls, 0);
302 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
305 /* Open database and precompile statements */
306 if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
308 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
310 _ ("Unable to initialize SQLite: %s.\n"),
311 sqlite3_errmsg (plugin->dbh));
312 return GNUNET_SYSERR;
316 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
319 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
320 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
321 "PRAGMA legacy_file_format=OFF",
325 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
326 "PRAGMA auto_vacuum=INCREMENTAL",
330 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
331 "PRAGMA locking_mode=EXCLUSIVE",
337 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL));
339 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
342 /* We have to do it here, because otherwise precompiling SQL might fail */
344 sq_prepare (plugin->dbh,
345 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
348 /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
349 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
350 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
351 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
352 */if ((SQLITE_DONE == sqlite3_step (stmt)) &&
353 (SQLITE_OK != sqlite3_exec (plugin->dbh,
354 "CREATE TABLE gn091 ("
355 " repl INT4 NOT NULL DEFAULT 0,"
356 " type INT4 NOT NULL DEFAULT 0,"
357 " prio INT4 NOT NULL DEFAULT 0,"
358 " anonLevel INT4 NOT NULL DEFAULT 0,"
359 " expire INT8 NOT NULL DEFAULT 0,"
360 " rvalue INT8 NOT NULL,"
361 " hash TEXT NOT NULL DEFAULT '',"
362 " vhash TEXT NOT NULL DEFAULT '',"
363 " value BLOB NOT NULL DEFAULT '')",
368 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
369 sqlite3_finalize (stmt);
370 return GNUNET_SYSERR;
372 sqlite3_finalize (stmt);
373 create_indices (plugin->dbh);
375 #define RESULT_COLUMNS \
376 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
378 (SQLITE_OK != sq_prepare (plugin->dbh,
380 "SET prio = prio + ?, "
382 "expire = MAX(expire, ?) "
383 "WHERE hash = ? AND vhash = ?",
385 (SQLITE_OK != sq_prepare (plugin->dbh,
387 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
388 &plugin->updRepl)) ||
389 (SQLITE_OK != sq_prepare (plugin->dbh,
390 "SELECT " RESULT_COLUMNS " FROM gn091 "
393 " NOT EXISTS (SELECT 1 FROM gn091 "
394 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
395 "ORDER BY rvalue ASC LIMIT 1",
396 &plugin->selRepl)) ||
397 (SQLITE_OK != sq_prepare (plugin->dbh,
398 "SELECT MAX(repl) FROM gn091",
399 &plugin->maxRepl)) ||
401 sq_prepare (plugin->dbh,
402 "SELECT " RESULT_COLUMNS " FROM gn091 "
403 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
404 "ORDER BY expire ASC LIMIT 1",
405 &plugin->selExpi)) ||
406 (SQLITE_OK != sq_prepare (plugin->dbh,
407 "SELECT " RESULT_COLUMNS " FROM gn091 "
408 "WHERE _ROWID_ >= ? AND "
411 "ORDER BY _ROWID_ ASC LIMIT 1",
412 &plugin->selZeroAnon)) ||
414 sq_prepare (plugin->dbh,
415 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
416 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
417 &plugin->insertContent)) ||
418 (SQLITE_OK != sq_prepare (plugin->dbh,
419 "SELECT " RESULT_COLUMNS " FROM gn091 "
420 "WHERE _ROWID_ >= ?1 "
421 "ORDER BY _ROWID_ ASC LIMIT 1",
423 (SQLITE_OK != sq_prepare (plugin->dbh,
424 "SELECT " RESULT_COLUMNS " FROM gn091 "
425 "WHERE _ROWID_ >= ?1 AND "
427 "ORDER BY _ROWID_ ASC LIMIT 1",
429 (SQLITE_OK != sq_prepare (plugin->dbh,
430 "SELECT " RESULT_COLUMNS " FROM gn091 "
431 "WHERE _ROWID_ >= ?1 AND "
433 "ORDER BY _ROWID_ ASC LIMIT 1",
435 (SQLITE_OK != sq_prepare (plugin->dbh,
436 "SELECT " RESULT_COLUMNS " FROM gn091 "
437 "WHERE _ROWID_ >= ?1 AND "
440 "ORDER BY _ROWID_ ASC LIMIT 1",
442 (SQLITE_OK != sq_prepare (plugin->dbh,
443 "SELECT " RESULT_COLUMNS " FROM gn091 "
444 "WHERE _ROWID_ >= ?1 AND "
446 "ORDER BY _ROWID_ ASC LIMIT 1",
448 (SQLITE_OK != sq_prepare (plugin->dbh,
449 "SELECT " RESULT_COLUMNS " FROM gn091 "
450 "WHERE _ROWID_ >= ?1 AND "
453 "ORDER BY _ROWID_ ASC LIMIT 1",
455 (SQLITE_OK != sq_prepare (plugin->dbh,
456 "SELECT " RESULT_COLUMNS " FROM gn091 "
457 "WHERE _ROWID_ >= ?1 AND "
460 "ORDER BY _ROWID_ ASC LIMIT 1",
462 (SQLITE_OK != sq_prepare (plugin->dbh,
463 "SELECT " RESULT_COLUMNS " FROM gn091 "
464 "WHERE _ROWID_ >= ?1 AND "
468 "ORDER BY _ROWID_ ASC LIMIT 1",
470 (SQLITE_OK != sq_prepare (plugin->dbh,
471 "DELETE FROM gn091 WHERE _ROWID_ = ?",
473 (SQLITE_OK != sq_prepare (plugin->dbh,
475 "WHERE hash = ? AND "
480 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
481 return GNUNET_SYSERR;
488 * Shutdown database connection and associate data
491 * @param plugin the plugin context (state for this module)
494 database_shutdown (struct Plugin *plugin)
498 #if SQLITE_VERSION_NUMBER >= 3007000
502 if (NULL != plugin->remove)
503 sqlite3_finalize (plugin->remove);
504 if (NULL != plugin->delRow)
505 sqlite3_finalize (plugin->delRow);
506 if (NULL != plugin->update)
507 sqlite3_finalize (plugin->update);
508 if (NULL != plugin->updRepl)
509 sqlite3_finalize (plugin->updRepl);
510 if (NULL != plugin->selRepl)
511 sqlite3_finalize (plugin->selRepl);
512 if (NULL != plugin->maxRepl)
513 sqlite3_finalize (plugin->maxRepl);
514 if (NULL != plugin->selExpi)
515 sqlite3_finalize (plugin->selExpi);
516 if (NULL != plugin->selZeroAnon)
517 sqlite3_finalize (plugin->selZeroAnon);
518 if (NULL != plugin->insertContent)
519 sqlite3_finalize (plugin->insertContent);
520 for (int i = 0; i < 8; ++i)
521 if (NULL != plugin->get[i])
522 sqlite3_finalize (plugin->get[i]);
523 result = sqlite3_close (plugin->dbh);
524 #if SQLITE_VERSION_NUMBER >= 3007000
525 if (result == SQLITE_BUSY)
528 GNUNET_ERROR_TYPE_WARNING,
531 "Tried to close sqlite without finalizing all prepared statements.\n"));
532 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
535 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
537 "Closing statement %p\n",
539 result = sqlite3_finalize (stmt);
540 if (result != SQLITE_OK)
541 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
543 "Failed to close statement %p: %d\n",
546 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
548 result = sqlite3_close (plugin->dbh);
551 if (SQLITE_OK != result)
552 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
553 GNUNET_free_non_null (plugin->fn);
558 * Delete the database entry with the given
561 * @param plugin the plugin context (state for this module)
562 * @param rid the ID of the row to delete
565 delete_by_rowid (struct Plugin *plugin, uint64_t rid)
567 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid),
568 GNUNET_SQ_query_param_end };
570 if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params))
571 return GNUNET_SYSERR;
572 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
575 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
577 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
578 return GNUNET_SYSERR;
580 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
586 * Store an item in the datastore.
589 * @param key key for the item
590 * @param absent true if the key was not found in the bloom filter
591 * @param size number of bytes in @a data
592 * @param data content stored
593 * @param type type of the content
594 * @param priority priority of the content
595 * @param anonymity anonymity-level for the content
596 * @param replication replication-level for the content
597 * @param expiration expiration time for the content
598 * @param cont continuation called with success or failure status
599 * @param cont_cls continuation closure
602 sqlite_plugin_put (void *cls,
603 const struct GNUNET_HashCode *key,
607 enum GNUNET_BLOCK_Type type,
610 uint32_t replication,
611 struct GNUNET_TIME_Absolute expiration,
615 struct Plugin *plugin = cls;
616 struct GNUNET_HashCode vhash;
619 GNUNET_CRYPTO_hash (data, size, &vhash);
623 struct GNUNET_SQ_QueryParam params[] =
624 { GNUNET_SQ_query_param_uint32 (&priority),
625 GNUNET_SQ_query_param_uint32 (&replication),
626 GNUNET_SQ_query_param_absolute_time (&expiration),
627 GNUNET_SQ_query_param_auto_from_type (key),
628 GNUNET_SQ_query_param_auto_from_type (&vhash),
629 GNUNET_SQ_query_param_end };
631 if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params))
633 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure"));
636 if (SQLITE_DONE != sqlite3_step (plugin->update))
638 LOG_SQLITE_MSG (plugin,
640 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
642 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
643 GNUNET_free_non_null (msg);
646 int changes = sqlite3_changes (plugin->dbh);
647 GNUNET_SQ_reset (plugin->dbh, plugin->update);
650 cont (cont_cls, key, size, GNUNET_NO, NULL);
656 uint32_t type32 = (uint32_t) type;
657 struct GNUNET_SQ_QueryParam params[] =
658 { GNUNET_SQ_query_param_uint32 (&replication),
659 GNUNET_SQ_query_param_uint32 (&type32),
660 GNUNET_SQ_query_param_uint32 (&priority),
661 GNUNET_SQ_query_param_uint32 (&anonymity),
662 GNUNET_SQ_query_param_absolute_time (&expiration),
663 GNUNET_SQ_query_param_uint64 (&rvalue),
664 GNUNET_SQ_query_param_auto_from_type (key),
665 GNUNET_SQ_query_param_auto_from_type (&vhash),
666 GNUNET_SQ_query_param_fixed_size (data, size),
667 GNUNET_SQ_query_param_end };
672 if (size > MAX_ITEM_SIZE)
674 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
678 GNUNET_ERROR_TYPE_DEBUG,
680 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
684 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
687 GNUNET_STRINGS_absolute_time_to_string (expiration));
688 stmt = plugin->insertContent;
689 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
690 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
692 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
695 n = sqlite3_step (stmt);
699 if (NULL != plugin->env->duc)
700 plugin->env->duc (plugin->env->cls,
701 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
702 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
704 "Stored new entry (%u bytes)\n",
705 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
711 LOG_SQLITE_MSG (plugin,
713 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
719 LOG_SQLITE_MSG (plugin,
721 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
723 GNUNET_SQ_reset (plugin->dbh, stmt);
724 database_shutdown (plugin);
725 database_setup (plugin->env->cfg, plugin);
726 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
727 GNUNET_free_non_null (msg);
730 GNUNET_SQ_reset (plugin->dbh, stmt);
731 cont (cont_cls, key, size, ret, msg);
732 GNUNET_free_non_null (msg);
737 * Execute statement that gets a row and call the callback
738 * with the result. Resets the statement afterwards.
740 * @param plugin the plugin
741 * @param stmt the statement
742 * @param proc processor to call
743 * @param proc_cls closure for @a proc
746 execute_get (struct Plugin *plugin,
748 PluginDatumProcessor proc,
752 struct GNUNET_TIME_Absolute expiration;
753 uint32_t replication;
760 struct GNUNET_HashCode key;
762 struct GNUNET_SQ_ResultSpec rs[] =
763 { GNUNET_SQ_result_spec_uint32 (&replication),
764 GNUNET_SQ_result_spec_uint32 (&type),
765 GNUNET_SQ_result_spec_uint32 (&priority),
766 GNUNET_SQ_result_spec_uint32 (&anonymity),
767 GNUNET_SQ_result_spec_absolute_time (&expiration),
768 GNUNET_SQ_result_spec_auto_from_type (&key),
769 GNUNET_SQ_result_spec_variable_size (&value, &value_size),
770 GNUNET_SQ_result_spec_uint64 (&rowid),
771 GNUNET_SQ_result_spec_end };
773 n = sqlite3_step (stmt);
777 if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs))
782 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
784 "Found reply in database with expiration %s\n",
785 GNUNET_STRINGS_absolute_time_to_string (expiration));
786 ret = proc (proc_cls,
796 GNUNET_SQ_cleanup_result (rs);
797 GNUNET_SQ_reset (plugin->dbh, stmt);
798 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
799 (NULL != plugin->env->duc))
800 plugin->env->duc (plugin->env->cls,
801 -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
805 /* database must be empty */
813 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
815 if (SQLITE_OK != sqlite3_reset (stmt))
817 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
820 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
821 database_shutdown (plugin);
822 database_setup (plugin->env->cfg, plugin);
825 GNUNET_SQ_reset (plugin->dbh, stmt);
826 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
831 * Select a subset of the items in the datastore and call
832 * the given processor for the item.
834 * @param cls our plugin context
835 * @param next_uid return the result with lowest uid >= next_uid
836 * @param type entries of which type should be considered?
837 * Must not be zero (ANY).
838 * @param proc function to call on the matching value;
839 * will be called with NULL if no value matches
840 * @param proc_cls closure for @a proc
843 sqlite_plugin_get_zero_anonymity (void *cls,
845 enum GNUNET_BLOCK_Type type,
846 PluginDatumProcessor proc,
849 struct Plugin *plugin = cls;
850 uint32_t type32 = type;
851 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (
853 GNUNET_SQ_query_param_uint32 (
855 GNUNET_SQ_query_param_end };
857 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
858 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params))
860 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
863 execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
868 * Get results for a particular key in the datastore.
871 * @param next_uid return the result with lowest uid >= next_uid
872 * @param random if true, return a random result instead of using next_uid
873 * @param key maybe NULL (to match all entries)
874 * @param type entries of which type are relevant?
875 * Use 0 for any type.
876 * @param proc function to call on the matching value;
877 * will be called with NULL if nothing matches
878 * @param proc_cls closure for @a proc
881 sqlite_plugin_get_key (void *cls,
884 const struct GNUNET_HashCode *key,
885 enum GNUNET_BLOCK_Type type,
886 PluginDatumProcessor proc,
889 struct Plugin *plugin = cls;
891 int use_rvalue = random;
892 uint32_t type32 = (uint32_t) type;
893 int use_type = GNUNET_BLOCK_TYPE_ANY != type;
894 int use_key = NULL != key;
895 sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type];
896 struct GNUNET_SQ_QueryParam params[] =
897 { GNUNET_SQ_query_param_uint64 (&next_uid),
898 GNUNET_SQ_query_param_uint64 (&rvalue),
899 GNUNET_SQ_query_param_auto_from_type (key),
900 GNUNET_SQ_query_param_uint32 (&type32),
901 GNUNET_SQ_query_param_end };
903 /* SQLite doesn't like it when you try to bind a parameter greater than the
904 * last numbered parameter, but unused parameters in the middle are OK.
908 params[3] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
911 params[2] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
913 params[1] = (struct GNUNET_SQ_QueryParam) GNUNET_SQ_query_param_end;
918 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
924 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
926 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
929 execute_get (plugin, stmt, proc, proc_cls);
934 * Context for #repl_proc() function.
939 * Function to call for the result (or the NULL).
941 PluginDatumProcessor proc;
944 * Closure for @e proc.
954 * Yes if UID was set.
961 * Wrapper for the processor for #sqlite_plugin_get_replication().
962 * Decrements the replication counter and calls the original
966 * @param key key for the content
967 * @param size number of bytes in @a data
968 * @param data content stored
969 * @param type type of the content
970 * @param priority priority of the content
971 * @param anonymity anonymity-level for the content
972 * @param replication replication-level for the content
973 * @param expiration expiration time for the content
974 * @param uid unique identifier for the datum;
975 * maybe 0 if no unique identifier is available
976 * @return #GNUNET_OK for normal return,
977 * #GNUNET_NO to delete the item
980 repl_proc (void *cls,
981 const struct GNUNET_HashCode *key,
984 enum GNUNET_BLOCK_Type type,
987 uint32_t replication,
988 struct GNUNET_TIME_Absolute expiration,
991 struct ReplCtx *rc = cls;
994 if (GNUNET_SYSERR == rc->have_uid)
995 rc->have_uid = GNUNET_NO;
996 ret = rc->proc (rc->proc_cls,
1009 rc->have_uid = GNUNET_YES;
1016 * Get a random item for replication. Returns a single random item
1017 * from those with the highest replication counters. The item's
1018 * replication counter is decremented by one IF it was positive before.
1019 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1021 * @param cls closure
1022 * @param proc function to call the value (once only).
1023 * @param proc_cls closure for @a proc
1026 sqlite_plugin_get_replication (void *cls,
1027 PluginDatumProcessor proc,
1030 struct Plugin *plugin = cls;
1034 struct GNUNET_SQ_QueryParam params_sel_repl[] =
1035 { GNUNET_SQ_query_param_uint64 (&rvalue),
1036 GNUNET_SQ_query_param_uint32 (&repl),
1037 GNUNET_SQ_query_param_end };
1038 struct GNUNET_SQ_QueryParam params_upd_repl[] =
1039 { GNUNET_SQ_query_param_uint64 (&rc.uid), GNUNET_SQ_query_param_end };
1041 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1043 "Getting random block based on replication order.\n");
1044 if (SQLITE_ROW != sqlite3_step (plugin->maxRepl))
1046 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1048 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1051 repl = sqlite3_column_int (plugin->maxRepl, 0);
1052 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1053 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
1054 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl))
1056 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1059 rc.have_uid = GNUNET_SYSERR;
1061 rc.proc_cls = proc_cls;
1062 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1063 if (GNUNET_YES == rc.have_uid)
1065 if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl))
1067 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1070 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1072 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1074 GNUNET_SQ_reset (plugin->dbh, plugin->updRepl);
1076 if (GNUNET_SYSERR == rc.have_uid)
1078 /* proc was not called at all so far, do it now. */
1079 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1085 * Get a random item that has expired or has low priority.
1086 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1088 * @param cls closure
1089 * @param proc function to call the value (once only).
1090 * @param proc_cls closure for @a proc
1093 sqlite_plugin_get_expiration (void *cls,
1094 PluginDatumProcessor proc,
1097 struct Plugin *plugin = cls;
1099 struct GNUNET_TIME_Absolute now;
1100 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_absolute_time (
1102 GNUNET_SQ_query_param_end };
1105 GNUNET_ERROR_TYPE_DEBUG,
1107 "Getting random block based on expiration and priority order.\n");
1108 now = GNUNET_TIME_absolute_get ();
1109 stmt = plugin->selExpi;
1110 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
1112 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1115 execute_get (plugin, stmt, proc, proc_cls);
1120 * Get all of the keys in the datastore.
1122 * @param cls closure
1123 * @param proc function to call on each key
1124 * @param proc_cls closure for @a proc
1127 sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls)
1129 struct Plugin *plugin = cls;
1130 struct GNUNET_HashCode key;
1131 struct GNUNET_SQ_ResultSpec results[] =
1132 { GNUNET_SQ_result_spec_auto_from_type (&key), GNUNET_SQ_result_spec_end };
1136 GNUNET_assert (NULL != proc);
1137 if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt))
1140 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1142 proc (proc_cls, NULL, 0);
1145 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1147 if (GNUNET_OK == GNUNET_SQ_extract_result (stmt, results))
1148 proc (proc_cls, &key, 1);
1152 if (SQLITE_DONE != ret)
1153 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1154 sqlite3_finalize (stmt);
1155 proc (proc_cls, NULL, 0);
1162 * @param cls our plugin context
1165 sqlite_plugin_drop (void *cls)
1167 struct Plugin *plugin = cls;
1169 plugin->drop_on_shutdown = GNUNET_YES;
1174 * Remove a particular key in the datastore.
1176 * @param cls closure
1177 * @param key key for the content
1178 * @param size number of bytes in data
1179 * @param data content stored
1180 * @param cont continuation called with success or failure status
1181 * @param cont_cls continuation closure for @a cont
1184 sqlite_plugin_remove_key (void *cls,
1185 const struct GNUNET_HashCode *key,
1188 PluginRemoveCont cont,
1191 struct Plugin *plugin = cls;
1192 struct GNUNET_SQ_QueryParam params[] =
1193 { GNUNET_SQ_query_param_auto_from_type (key),
1194 GNUNET_SQ_query_param_fixed_size (data, size),
1195 GNUNET_SQ_query_param_end };
1197 if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params))
1199 cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed");
1202 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1205 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1207 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1208 cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed");
1211 int changes = sqlite3_changes (plugin->dbh);
1212 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1215 cont (cont_cls, key, size, GNUNET_NO, NULL);
1218 if (NULL != plugin->env->duc)
1219 plugin->env->duc (plugin->env->cls,
1220 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
1221 cont (cont_cls, key, size, GNUNET_OK, NULL);
1226 * Get an estimate of how much space the database is
1229 * @param cls the `struct Plugin`
1230 * @return the size of the database on disk (estimate)
1233 sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
1235 struct Plugin *plugin = cls;
1244 if (NULL == estimate)
1246 if (SQLITE_VERSION_NUMBER < 3006000)
1249 GNUNET_ERROR_TYPE_WARNING,
1251 _ ("sqlite version to old to determine size, assuming zero\n"));
1255 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1256 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
1257 "PRAGMA auto_vacuum=INCREMENTAL",
1261 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt));
1262 if (SQLITE_ROW == sqlite3_step (stmt))
1263 pages = sqlite3_column_int64 (stmt, 0);
1266 sqlite3_finalize (stmt);
1267 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt));
1268 CHECK (SQLITE_ROW == sqlite3_step (stmt));
1269 page_size = sqlite3_column_int64 (stmt, 0);
1270 sqlite3_finalize (stmt);
1272 GNUNET_ERROR_TYPE_INFO,
1274 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1275 (unsigned long long) pages,
1276 (unsigned long long) page_size);
1277 *estimate = pages * page_size;
1282 * Entry point for the plugin.
1284 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1285 * @return NULL on error, othrewise the plugin context
1288 libgnunet_plugin_datastore_sqlite_init (void *cls)
1290 static struct Plugin plugin;
1291 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1292 struct GNUNET_DATASTORE_PluginFunctions *api;
1294 if (NULL != plugin.env)
1295 return NULL; /* can only initialize once! */
1296 memset (&plugin, 0, sizeof(struct Plugin));
1298 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1300 database_shutdown (&plugin);
1303 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1305 api->estimate_size = &sqlite_plugin_estimate_size;
1306 api->put = &sqlite_plugin_put;
1307 api->get_key = &sqlite_plugin_get_key;
1308 api->get_replication = &sqlite_plugin_get_replication;
1309 api->get_expiration = &sqlite_plugin_get_expiration;
1310 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1311 api->get_keys = &sqlite_plugin_get_keys;
1312 api->drop = &sqlite_plugin_drop;
1313 api->remove_key = &sqlite_plugin_remove_key;
1314 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1316 _ ("Sqlite database running\n"));
1322 * Exit point from the plugin.
1324 * @param cls the plugin context (as returned by "init")
1325 * @return always NULL
1328 libgnunet_plugin_datastore_sqlite_done (void *cls)
1331 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1332 struct Plugin *plugin = api->cls;
1334 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1336 "sqlite plugin is done\n");
1338 if (plugin->drop_on_shutdown)
1339 fn = GNUNET_strdup (plugin->fn);
1340 database_shutdown (plugin);
1345 if (0 != unlink (fn))
1346 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1353 /* end of plugin_datastore_sqlite.c */