2 * This file is part of GNUnet
3 * (C) 2009, 2011 Christian Grothoff (and other contributing authors)
5 * GNUnet is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published
7 * by the Free Software Foundation; either version 3, or (at your
8 * option) any later version.
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with GNUnet; see the file COPYING. If not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
28 #include "gnunet_datastore_plugin.h"
33 * We allocate items on the stack at times. To prevent a stack
34 * overflow, we impose a limit on the maximum size for the data per
35 * item. 64k should be enough.
37 #define MAX_ITEM_SIZE 65536
40 * After how many ms "busy" should a DB operation fail for good?
41 * A low value makes sure that we are more responsive to requests
42 * (especially PUTs). A high value guarantees a higher success
43 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
45 * The default value of 250ms should ensure that users do not experience
46 * huge latencies while at the same time allowing operations to succeed
47 * with reasonable probability.
49 #define BUSY_TIMEOUT_MS 250
53 * Log an error message at log-level 'level' that indicates
54 * a failure of the command 'cmd' on file 'filename'
55 * with the message given by strerror(errno).
57 #define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed at %s:%u with error: %s"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); } while(0)
62 * Context for all functions in this plugin.
67 * Our execution environment.
69 struct GNUNET_DATASTORE_PluginEnvironment *env;
77 * Native SQLite database handle.
82 * Precompiled SQL for deletion.
87 * Precompiled SQL for update.
89 sqlite3_stmt *updPrio;
92 * Get maximum repl value in database.
94 sqlite3_stmt *maxRepl;
97 * Precompiled SQL for replication decrement.
99 sqlite3_stmt *updRepl;
102 * Precompiled SQL for replication selection.
104 sqlite3_stmt *selRepl;
107 * Precompiled SQL for expiration selection.
109 sqlite3_stmt *selExpi;
112 * Precompiled SQL for expiration selection.
114 sqlite3_stmt *selZeroAnon;
117 * Precompiled SQL for insertion.
119 sqlite3_stmt *insertContent;
122 * Should the database be dropped on shutdown?
124 int drop_on_shutdown;
130 * @brief Prepare a SQL statement
132 * @param dbh handle to the database
133 * @param zSql SQL statement, UTF-8 encoded
134 * @param ppStmt set to the prepared statement
135 * @return 0 on success
138 sq_prepare (sqlite3 *dbh,
140 sqlite3_stmt **ppStmt)
146 sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
147 (const char **) &dummy);
148 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
149 "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
155 * Create our database indices.
157 * @param dbh handle to the database
160 create_indices (sqlite3 * dbh)
164 sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)",
165 NULL, NULL, NULL)) ||
168 "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)",
169 NULL, NULL, NULL)) ||
172 "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)",
173 NULL, NULL, NULL)) ||
176 "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)",
177 NULL, NULL, NULL)) ||
180 "CREATE INDEX IF NOT EXISTS idx_anon_type_hash ON gn090 (anonLevel ASC,type,hash)",
181 NULL, NULL, NULL)) ||
184 "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire ASC)",
185 NULL, NULL, NULL)) ||
188 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)",
189 NULL, NULL, NULL)) ||
192 "CREATE INDEX IF NOT EXISTS idx_repl ON gn090 (repl DESC)",
194 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
195 "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
200 #define CHECK(a) GNUNET_break(a)
204 #define ENULL_DEFINED 1
205 #define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "%s\n", e); sqlite3_free(e); }
210 * Initialize the database connections and associated
211 * data structures (create tables and indices
212 * as needed as well).
214 * @param cfg our configuration
215 * @param plugin the plugin context (state for this module)
216 * @return #GNUNET_OK on success
219 database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
220 struct Plugin *plugin)
230 GNUNET_CONFIGURATION_get_value_filename (cfg, "datastore-sqlite",
231 "FILENAME", &afsdir))
233 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
234 "datastore-sqlite", "FILENAME");
235 return GNUNET_SYSERR;
237 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
239 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
242 GNUNET_free (afsdir);
243 return GNUNET_SYSERR;
245 /* database is new or got deleted, reset payload to zero! */
246 plugin->env->duc (plugin->env->cls, 0);
248 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
251 /* Open database and precompile statements */
252 if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK)
254 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
255 _("Unable to initialize SQLite: %s.\n"),
256 sqlite3_errmsg (plugin->dbh));
257 return GNUNET_SYSERR;
260 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
263 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL,
266 sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL,
269 sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
272 sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
275 sqlite3_exec (plugin->dbh, "PRAGMA count_changes=OFF", NULL, NULL,
278 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
281 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
284 /* We have to do it here, because otherwise precompiling SQL might fail */
286 sq_prepare (plugin->dbh,
287 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn090'",
289 if ((sqlite3_step (stmt) == SQLITE_DONE) &&
292 "CREATE TABLE gn090 (" " repl INT4 NOT NULL DEFAULT 0,"
293 " type INT4 NOT NULL DEFAULT 0," " prio INT4 NOT NULL DEFAULT 0,"
294 " anonLevel INT4 NOT NULL DEFAULT 0,"
295 " expire INT8 NOT NULL DEFAULT 0," " rvalue INT8 NOT NULL,"
296 " hash TEXT NOT NULL DEFAULT ''," " vhash TEXT NOT NULL DEFAULT '',"
297 " value BLOB NOT NULL DEFAULT '')", NULL, NULL, NULL) != SQLITE_OK))
299 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
300 sqlite3_finalize (stmt);
301 return GNUNET_SYSERR;
303 sqlite3_finalize (stmt);
304 create_indices (plugin->dbh);
309 "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
310 &plugin->updPrio) != SQLITE_OK) ||
313 "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
314 &plugin->updRepl) != SQLITE_OK) ||
317 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
318 #if SQLITE_VERSION_NUMBER >= 3007000
319 "INDEXED BY idx_repl_rvalue "
321 "WHERE repl=?2 AND " " (rvalue>=?1 OR "
322 " NOT EXISTS (SELECT 1 FROM gn090 "
323 #if SQLITE_VERSION_NUMBER >= 3007000
324 "INDEXED BY idx_repl_rvalue "
326 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
327 "ORDER BY rvalue ASC LIMIT 1", &plugin->selRepl) != SQLITE_OK) ||
328 (sq_prepare (plugin->dbh, "SELECT MAX(repl) FROM gn090"
329 #if SQLITE_VERSION_NUMBER >= 3007000
330 " INDEXED BY idx_repl_rvalue"
332 "", &plugin->maxRepl) != SQLITE_OK) ||
335 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
336 #if SQLITE_VERSION_NUMBER >= 3007000
337 "INDEXED BY idx_expire "
339 "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
340 "ORDER BY expire ASC LIMIT 1", &plugin->selExpi) != SQLITE_OK) ||
343 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
344 #if SQLITE_VERSION_NUMBER >= 3007000
345 "INDEXED BY idx_anon_type_hash "
347 "WHERE (anonLevel = 0 AND type=?1) "
348 "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
349 &plugin->selZeroAnon) != SQLITE_OK) ||
352 "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
353 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
354 &plugin->insertContent) != SQLITE_OK) ||
356 (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?",
357 &plugin->delRow) != SQLITE_OK))
359 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "precompiling");
360 return GNUNET_SYSERR;
368 * Shutdown database connection and associate data
371 * @param plugin the plugin context (state for this module)
374 database_shutdown (struct Plugin *plugin)
378 #if SQLITE_VERSION_NUMBER >= 3007000
382 if (plugin->delRow != NULL)
383 sqlite3_finalize (plugin->delRow);
384 if (plugin->updPrio != NULL)
385 sqlite3_finalize (plugin->updPrio);
386 if (plugin->updRepl != NULL)
387 sqlite3_finalize (plugin->updRepl);
388 if (plugin->selRepl != NULL)
389 sqlite3_finalize (plugin->selRepl);
390 if (plugin->maxRepl != NULL)
391 sqlite3_finalize (plugin->maxRepl);
392 if (plugin->selExpi != NULL)
393 sqlite3_finalize (plugin->selExpi);
394 if (plugin->selZeroAnon != NULL)
395 sqlite3_finalize (plugin->selZeroAnon);
396 if (plugin->insertContent != NULL)
397 sqlite3_finalize (plugin->insertContent);
398 result = sqlite3_close (plugin->dbh);
399 #if SQLITE_VERSION_NUMBER >= 3007000
400 if (result == SQLITE_BUSY)
402 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
404 ("Tried to close sqlite without finalizing all prepared statements.\n"));
405 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
408 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
409 "Closing statement %p\n", stmt);
410 result = sqlite3_finalize (stmt);
411 if (result != SQLITE_OK)
412 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
413 "Failed to close statement %p: %d\n", stmt, result);
414 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
416 result = sqlite3_close (plugin->dbh);
419 if (SQLITE_OK != result)
420 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
422 GNUNET_free_non_null (plugin->fn);
427 * Delete the database entry with the given
430 * @param plugin the plugin context (state for this module)
431 * @param rid the ID of the row to delete
434 delete_by_rowid (struct Plugin *plugin,
435 unsigned long long rid)
437 if (SQLITE_OK != sqlite3_bind_int64 (plugin->delRow, 1, rid))
439 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
440 "sqlite3_bind_XXXX");
441 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
442 LOG_SQLITE (plugin, NULL,
443 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
445 return GNUNET_SYSERR;
447 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
449 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
451 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
452 LOG_SQLITE (plugin, NULL,
453 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
455 return GNUNET_SYSERR;
457 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
458 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
465 * Store an item in the datastore.
468 * @param key key for the item
469 * @param size number of bytes in @a data
470 * @param data content stored
471 * @param type type of the content
472 * @param priority priority of the content
473 * @param anonymity anonymity-level for the content
474 * @param replication replication-level for the content
475 * @param expiration expiration time for the content
476 * @param msg set to an error message
477 * @return #GNUNET_OK on success
480 sqlite_plugin_put (void *cls,
481 const struct GNUNET_HashCode *key,
484 enum GNUNET_BLOCK_Type type,
487 uint32_t replication,
488 struct GNUNET_TIME_Absolute expiration,
491 struct Plugin *plugin = cls;
495 struct GNUNET_HashCode vhash;
498 if (size > MAX_ITEM_SIZE)
499 return GNUNET_SYSERR;
500 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
501 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
502 type, GNUNET_h2s (key), priority,
504 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (expiration),
506 GNUNET_STRINGS_absolute_time_to_string (expiration));
507 GNUNET_CRYPTO_hash (data, size, &vhash);
508 stmt = plugin->insertContent;
509 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
510 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) ||
511 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
512 (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
513 (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
514 (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value_us)) ||
515 (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) ||
517 sqlite3_bind_blob (stmt, 7, key, sizeof (struct GNUNET_HashCode),
518 SQLITE_TRANSIENT)) ||
520 sqlite3_bind_blob (stmt, 8, &vhash, sizeof (struct GNUNET_HashCode),
521 SQLITE_TRANSIENT)) ||
522 (SQLITE_OK != sqlite3_bind_blob (stmt, 9, data, size, SQLITE_TRANSIENT)))
524 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
525 "sqlite3_bind_XXXX");
526 if (SQLITE_OK != sqlite3_reset (stmt))
527 LOG_SQLITE (plugin, NULL,
528 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
530 return GNUNET_SYSERR;
532 n = sqlite3_step (stmt);
536 plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
537 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
538 "Stored new entry (%u bytes)\n",
539 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
544 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
549 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
551 if (SQLITE_OK != sqlite3_reset (stmt))
552 LOG_SQLITE (plugin, NULL,
553 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
555 database_shutdown (plugin);
556 database_setup (plugin->env->cfg, plugin);
557 return GNUNET_SYSERR;
559 if (SQLITE_OK != sqlite3_reset (stmt))
560 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
567 * Update the priority for a particular key in the datastore. If
568 * the expiration time in value is different than the time found in
569 * the datastore, the higher value should be kept. For the
570 * anonymity level, the lower value is to be used. The specified
571 * priority should be added to the existing priority, ignoring the
574 * Note that it is possible for multiple values to match this put.
575 * In that case, all of the respective values are updated.
577 * @param cls the plugin context (state for this module)
578 * @param uid unique identifier of the datum
579 * @param delta by how much should the priority
580 * change? If priority + delta < 0 the
581 * priority should be set to 0 (never go
583 * @param expire new expiration time should be the
584 * MAX of any existing expiration time and
586 * @param msg set to an error message
587 * @return #GNUNET_OK on success
590 sqlite_plugin_update (void *cls,
593 struct GNUNET_TIME_Absolute expire,
596 struct Plugin *plugin = cls;
599 if ((SQLITE_OK != sqlite3_bind_int (plugin->updPrio, 1, delta)) ||
600 (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value_us))
601 || (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 3, uid)))
603 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
604 "sqlite3_bind_XXXX");
605 if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
606 LOG_SQLITE (plugin, NULL,
607 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
609 return GNUNET_SYSERR;
612 n = sqlite3_step (plugin->updPrio);
613 if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
614 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
619 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Block updated\n");
622 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
626 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
628 return GNUNET_SYSERR;
634 * Execute statement that gets a row and call the callback
635 * with the result. Resets the statement afterwards.
637 * @param plugin the plugin
638 * @param stmt the statement
639 * @param proc processor to call
640 * @param proc_cls closure for @a proc
643 execute_get (struct Plugin *plugin,
645 PluginDatumProcessor proc,
649 struct GNUNET_TIME_Absolute expiration;
650 unsigned long long rowid;
654 n = sqlite3_step (stmt);
658 size = sqlite3_column_bytes (stmt, 5);
659 rowid = sqlite3_column_int64 (stmt, 6);
660 if (sqlite3_column_bytes (stmt, 4) != sizeof (struct GNUNET_HashCode))
662 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
663 _("Invalid data in database. Trying to fix (by deletion).\n"));
664 if (SQLITE_OK != sqlite3_reset (stmt))
665 LOG_SQLITE (plugin, NULL,
666 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
668 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
669 plugin->env->duc (plugin->env->cls,
670 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
673 expiration.abs_value_us = sqlite3_column_int64 (stmt, 3);
674 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
675 "Found reply in database with expiration %s\n",
676 GNUNET_STRINGS_absolute_time_to_string (expiration));
677 ret = proc (proc_cls, sqlite3_column_blob (stmt, 4) /* key */ ,
678 size, sqlite3_column_blob (stmt, 5) /* data */ ,
679 sqlite3_column_int (stmt, 0) /* type */ ,
680 sqlite3_column_int (stmt, 1) /* priority */ ,
681 sqlite3_column_int (stmt, 2) /* anonymity */ ,
683 if (SQLITE_OK != sqlite3_reset (stmt))
684 LOG_SQLITE (plugin, NULL,
685 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
687 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)))
688 plugin->env->duc (plugin->env->cls,
689 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
692 /* database must be empty */
693 if (SQLITE_OK != sqlite3_reset (stmt))
694 LOG_SQLITE (plugin, NULL,
695 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
702 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
704 if (SQLITE_OK != sqlite3_reset (stmt))
705 LOG_SQLITE (plugin, NULL,
706 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
709 database_shutdown (plugin);
710 database_setup (plugin->env->cfg, plugin);
713 if (SQLITE_OK != sqlite3_reset (stmt))
714 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
716 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
721 * Select a subset of the items in the datastore and call
722 * the given processor for the item.
724 * @param cls our plugin context
725 * @param offset offset of the result (modulo num-results);
726 * specific ordering does not matter for the offset
727 * @param type entries of which type should be considered?
728 * Use 0 for any type.
729 * @param proc function to call on each matching value;
730 * will be called once with a NULL value at the end
731 * @param proc_cls closure for @a proc
734 sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
735 enum GNUNET_BLOCK_Type type,
736 PluginDatumProcessor proc, void *proc_cls)
738 struct Plugin *plugin = cls;
741 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
742 stmt = plugin->selZeroAnon;
743 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
744 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)))
746 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
747 "sqlite3_bind_XXXX");
748 if (SQLITE_OK != sqlite3_reset (stmt))
749 LOG_SQLITE (plugin, NULL,
750 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
752 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
755 execute_get (plugin, stmt, proc, proc_cls);
760 * Get results for a particular key in the datastore.
763 * @param offset offset (mod count).
764 * @param key key to match, never NULL
765 * @param vhash hash of the value, maybe NULL (to
766 * match all values that have the right key).
767 * Note that for DBlocks there is no difference
768 * betwen key and vhash, but for other blocks
770 * @param type entries of which type are relevant?
771 * Use 0 for any type.
772 * @param proc function to call on each matching value;
773 * will be called once with a NULL value at the end
774 * @param proc_cls closure for @a proc
777 sqlite_plugin_get_key (void *cls,
779 const struct GNUNET_HashCode *key,
780 const struct GNUNET_HashCode *vhash,
781 enum GNUNET_BLOCK_Type type,
782 PluginDatumProcessor proc,
785 struct Plugin *plugin = cls;
793 GNUNET_assert (proc != NULL);
794 GNUNET_assert (key != NULL);
795 GNUNET_snprintf (scratch, sizeof (scratch),
796 "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
797 vhash == NULL ? "" : " AND vhash=?",
798 type == 0 ? "" : " AND type=?");
799 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
801 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
803 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
808 sqlite3_bind_blob (stmt, sqoff++, key, sizeof (struct GNUNET_HashCode),
810 if ((vhash != NULL) && (ret == SQLITE_OK))
812 sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (struct GNUNET_HashCode),
814 if ((type != 0) && (ret == SQLITE_OK))
815 ret = sqlite3_bind_int (stmt, sqoff++, type);
816 if (SQLITE_OK != ret)
818 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
819 sqlite3_finalize (stmt);
820 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
823 ret = sqlite3_step (stmt);
824 if (ret != SQLITE_ROW)
826 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
828 sqlite3_finalize (stmt);
829 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
832 total = sqlite3_column_int (stmt, 0);
833 sqlite3_finalize (stmt);
836 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
839 limit_off = (int) (offset % total);
842 GNUNET_snprintf (scratch, sizeof (scratch),
843 "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
844 "FROM gn090 WHERE hash=?%s%s "
845 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
846 vhash == NULL ? "" : " AND vhash=?",
847 type == 0 ? "" : " AND type=?");
848 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
850 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
852 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
856 ret = sqlite3_bind_blob (stmt, sqoff++, key,
857 sizeof (struct GNUNET_HashCode),
859 if ((vhash != NULL) && (ret == SQLITE_OK))
860 ret = sqlite3_bind_blob (stmt, sqoff++, vhash,
861 sizeof (struct GNUNET_HashCode),
863 if ((type != 0) && (ret == SQLITE_OK))
864 ret = sqlite3_bind_int (stmt, sqoff++, type);
865 if (ret == SQLITE_OK)
866 ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
867 if (ret != SQLITE_OK)
869 LOG_SQLITE (plugin, NULL,
870 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
872 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
875 execute_get (plugin, stmt, proc, proc_cls);
876 sqlite3_finalize (stmt);
881 * Context for #repl_proc() function.
887 * Function to call for the result (or the NULL).
889 PluginDatumProcessor proc;
892 * Closure for @e proc.
902 * Yes if UID was set.
909 * Wrapper for the processor for #sqlite_plugin_get_replication().
910 * Decrements the replication counter and calls the original
914 * @param key key for the content
915 * @param size number of bytes in @a data
916 * @param data content stored
917 * @param type type of the content
918 * @param priority priority of the content
919 * @param anonymity anonymity-level for the content
920 * @param expiration expiration time for the content
921 * @param uid unique identifier for the datum;
922 * maybe 0 if no unique identifier is available
923 * @return #GNUNET_OK for normal return,
924 * #GNUNET_NO to delete the item
927 repl_proc (void *cls,
928 const struct GNUNET_HashCode *key,
931 enum GNUNET_BLOCK_Type type,
934 struct GNUNET_TIME_Absolute expiration,
937 struct ReplCtx *rc = cls;
940 ret = rc->proc (rc->proc_cls,
950 rc->have_uid = GNUNET_YES;
957 * Get a random item for replication. Returns a single random item
958 * from those with the highest replication counters. The item's
959 * replication counter is decremented by one IF it was positive before.
960 * Call @a proc with all values ZERO or NULL if the datastore is empty.
963 * @param proc function to call the value (once only).
964 * @param proc_cls closure for @a proc
967 sqlite_plugin_get_replication (void *cls, PluginDatumProcessor proc,
970 struct Plugin *plugin = cls;
976 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
977 "Getting random block based on replication order.\n");
978 rc.have_uid = GNUNET_NO;
980 rc.proc_cls = proc_cls;
981 stmt = plugin->maxRepl;
982 if (SQLITE_ROW != sqlite3_step (stmt))
984 if (SQLITE_OK != sqlite3_reset (stmt))
985 LOG_SQLITE (plugin, NULL,
986 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
989 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
992 repl = sqlite3_column_int (stmt, 0);
993 if (SQLITE_OK != sqlite3_reset (stmt))
994 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
996 stmt = plugin->selRepl;
997 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
998 if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue))
1000 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1001 "sqlite3_bind_XXXX");
1002 if (SQLITE_OK != sqlite3_reset (stmt))
1003 LOG_SQLITE (plugin, NULL,
1004 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1006 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1009 if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl))
1011 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1012 "sqlite3_bind_XXXX");
1013 if (SQLITE_OK != sqlite3_reset (stmt))
1014 LOG_SQLITE (plugin, NULL,
1015 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1017 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1020 execute_get (plugin, stmt, &repl_proc, &rc);
1021 if (GNUNET_YES == rc.have_uid)
1023 if (SQLITE_OK != sqlite3_bind_int64 (plugin->updRepl, 1, rc.uid))
1025 LOG_SQLITE (plugin, NULL,
1026 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1027 "sqlite3_bind_XXXX");
1028 if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
1029 LOG_SQLITE (plugin, NULL,
1030 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1034 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1035 LOG_SQLITE (plugin, NULL,
1036 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1038 if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
1039 LOG_SQLITE (plugin, NULL,
1040 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1047 * Get a random item that has expired or has low priority.
1048 * Call @a proc with all values ZERO or NULL if the datastore is empty.
1050 * @param cls closure
1051 * @param proc function to call the value (once only).
1052 * @param proc_cls closure for @a proc
1055 sqlite_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
1058 struct Plugin *plugin = cls;
1060 struct GNUNET_TIME_Absolute now;
1062 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1063 "Getting random block based on expiration and priority order.\n");
1064 now = GNUNET_TIME_absolute_get ();
1065 stmt = plugin->selExpi;
1066 if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value_us))
1068 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1069 "sqlite3_bind_XXXX");
1070 if (SQLITE_OK != sqlite3_reset (stmt))
1071 LOG_SQLITE (plugin, NULL,
1072 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1074 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1077 execute_get (plugin, stmt, proc, proc_cls);
1082 * Get all of the keys in the datastore.
1084 * @param cls closure
1085 * @param proc function to call on each key
1086 * @param proc_cls closure for @a proc
1089 sqlite_plugin_get_keys (void *cls,
1090 PluginKeyProcessor proc,
1093 struct Plugin *plugin = cls;
1094 const struct GNUNET_HashCode *key;
1098 GNUNET_assert (proc != NULL);
1099 if (sq_prepare (plugin->dbh, "SELECT hash FROM gn090", &stmt) != SQLITE_OK)
1101 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1105 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1107 key = sqlite3_column_blob (stmt, 0);
1108 if (sizeof (struct GNUNET_HashCode) == sqlite3_column_bytes (stmt, 0))
1109 proc (proc_cls, key, 1);
1113 if (SQLITE_DONE != ret)
1114 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1115 sqlite3_finalize (stmt);
1122 * @param cls our plugin context
1125 sqlite_plugin_drop (void *cls)
1127 struct Plugin *plugin = cls;
1129 plugin->drop_on_shutdown = GNUNET_YES;
1134 * Get an estimate of how much space the database is
1137 * @param cls the `struct Plugin`
1138 * @return the size of the database on disk (estimate)
1140 static unsigned long long
1141 sqlite_plugin_estimate_size (void *cls)
1143 struct Plugin *plugin = cls;
1152 if (SQLITE_VERSION_NUMBER < 3006000)
1154 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "datastore-sqlite",
1156 ("sqlite version to old to determine size, assuming zero\n"));
1159 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1161 sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
1163 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt));
1164 if (SQLITE_ROW == sqlite3_step (stmt))
1165 pages = sqlite3_column_int64 (stmt, 0);
1168 sqlite3_finalize (stmt);
1169 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt));
1170 CHECK (SQLITE_ROW == sqlite3_step (stmt));
1171 page_size = sqlite3_column_int64 (stmt, 0);
1172 sqlite3_finalize (stmt);
1173 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1175 ("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1176 (unsigned long long) pages, (unsigned long long) page_size);
1177 return pages * page_size;
1182 * Entry point for the plugin.
1184 * @param cls the `struct GNUNET_DATASTORE_PluginEnvironment *`
1185 * @return NULL on error, othrewise the plugin context
1188 libgnunet_plugin_datastore_sqlite_init (void *cls)
1190 static struct Plugin plugin;
1191 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1192 struct GNUNET_DATASTORE_PluginFunctions *api;
1194 if (plugin.env != NULL)
1195 return NULL; /* can only initialize once! */
1196 memset (&plugin, 0, sizeof (struct Plugin));
1198 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1200 database_shutdown (&plugin);
1203 api = GNUNET_new (struct GNUNET_DATASTORE_PluginFunctions);
1205 api->estimate_size = &sqlite_plugin_estimate_size;
1206 api->put = &sqlite_plugin_put;
1207 api->update = &sqlite_plugin_update;
1208 api->get_key = &sqlite_plugin_get_key;
1209 api->get_replication = &sqlite_plugin_get_replication;
1210 api->get_expiration = &sqlite_plugin_get_expiration;
1211 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1212 api->get_keys = &sqlite_plugin_get_keys;
1213 api->drop = &sqlite_plugin_drop;
1214 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite",
1215 _("Sqlite database running\n"));
1221 * Exit point from the plugin.
1223 * @param cls the plugin context (as returned by "init")
1224 * @return always NULL
1227 libgnunet_plugin_datastore_sqlite_done (void *cls)
1230 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1231 struct Plugin *plugin = api->cls;
1233 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1234 "sqlite plugin is done\n");
1236 if (plugin->drop_on_shutdown)
1237 fn = GNUNET_strdup (plugin->fn);
1238 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1239 "Shutting down database\n");
1240 database_shutdown (plugin);
1245 if (0 != UNLINK (fn))
1246 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1249 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1250 "sqlite plugin is finished\n");
1254 /* end of plugin_datastore_sqlite.c */