2 * This file is part of GNUnet
3 * (C) 2009, 2011 Christian Grothoff (and other contributing authors)
5 * GNUnet is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published
7 * by the Free Software Foundation; either version 3, or (at your
8 * option) any later version.
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with GNUnet; see the file COPYING. If not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
28 #include "gnunet_datastore_plugin.h"
33 * We allocate items on the stack at times. To prevent a stack
34 * overflow, we impose a limit on the maximum size for the data per
35 * item. 64k should be enough.
37 #define MAX_ITEM_SIZE 65536
40 * After how many ms "busy" should a DB operation fail for good?
41 * A low value makes sure that we are more responsive to requests
42 * (especially PUTs). A high value guarantees a higher success
43 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
45 * The default value of 250ms should ensure that users do not experience
46 * huge latencies while at the same time allowing operations to succeed
47 * with reasonable probability.
49 #define BUSY_TIMEOUT_MS 250
53 * Log an error message at log-level 'level' that indicates
54 * a failure of the command 'cmd' on file 'filename'
55 * with the message given by strerror(errno).
57 #define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed at %s:%u with error: %s"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); } while(0)
62 * Context for all functions in this plugin.
67 * Our execution environment.
69 struct GNUNET_DATASTORE_PluginEnvironment *env;
77 * Native SQLite database handle.
82 * Precompiled SQL for deletion.
87 * Precompiled SQL for update.
89 sqlite3_stmt *updPrio;
92 * Get maximum repl value in database.
94 sqlite3_stmt *maxRepl;
97 * Precompiled SQL for replication decrement.
99 sqlite3_stmt *updRepl;
102 * Precompiled SQL for replication selection.
104 sqlite3_stmt *selRepl;
107 * Precompiled SQL for expiration selection.
109 sqlite3_stmt *selExpi;
112 * Precompiled SQL for expiration selection.
114 sqlite3_stmt *selZeroAnon;
117 * Precompiled SQL for insertion.
119 sqlite3_stmt *insertContent;
122 * Should the database be dropped on shutdown?
124 int drop_on_shutdown;
130 * @brief Prepare a SQL statement
132 * @param dbh handle to the database
133 * @param zSql SQL statement, UTF-8 encoded
134 * @param ppStmt set to the prepared statement
135 * @return 0 on success
138 sq_prepare (sqlite3 * dbh, const char *zSql, sqlite3_stmt ** ppStmt)
144 sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
145 (const char **) &dummy);
146 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
147 "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
153 * Create our database indices.
155 * @param dbh handle to the database
158 create_indices (sqlite3 * dbh)
162 sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)",
163 NULL, NULL, NULL)) ||
166 "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)",
167 NULL, NULL, NULL)) ||
170 "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)",
171 NULL, NULL, NULL)) ||
174 "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)",
175 NULL, NULL, NULL)) ||
178 "CREATE INDEX IF NOT EXISTS idx_anon_type_hash ON gn090 (anonLevel ASC,type,hash)",
179 NULL, NULL, NULL)) ||
182 "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire ASC)",
183 NULL, NULL, NULL)) ||
186 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)",
187 NULL, NULL, NULL)) ||
190 "CREATE INDEX IF NOT EXISTS idx_repl ON gn090 (repl DESC)",
192 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
193 "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
198 #define CHECK(a) GNUNET_break(a)
202 #define ENULL_DEFINED 1
203 #define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "%s\n", e); sqlite3_free(e); }
208 * Initialize the database connections and associated
209 * data structures (create tables and indices
210 * as needed as well).
212 * @param cfg our configuration
213 * @param plugin the plugin context (state for this module)
214 * @return GNUNET_OK on success
217 database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
218 struct Plugin *plugin)
228 GNUNET_CONFIGURATION_get_value_filename (cfg, "datastore-sqlite",
229 "FILENAME", &afsdir))
231 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
233 ("Option `%s' in section `%s' missing in configuration!\n"),
234 "FILENAME", "datastore-sqlite");
235 return GNUNET_SYSERR;
237 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
239 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
242 GNUNET_free (afsdir);
243 return GNUNET_SYSERR;
245 /* database is new or got deleted, reset payload to zero! */
246 plugin->env->duc (plugin->env->cls, 0);
250 GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), nl_langinfo (CODESET));
252 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), "UTF-8"); /* good luck */
254 GNUNET_free (afsdir);
256 /* Open database and precompile statements */
257 if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK)
259 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
260 _("Unable to initialize SQLite: %s.\n"),
261 sqlite3_errmsg (plugin->dbh));
262 return GNUNET_SYSERR;
265 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
268 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL,
271 sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL,
274 sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
277 sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
280 sqlite3_exec (plugin->dbh, "PRAGMA count_changes=OFF", NULL, NULL,
283 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
286 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
289 /* We have to do it here, because otherwise precompiling SQL might fail */
291 sq_prepare (plugin->dbh,
292 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn090'",
294 if ((sqlite3_step (stmt) == SQLITE_DONE) &&
297 "CREATE TABLE gn090 (" " repl INT4 NOT NULL DEFAULT 0,"
298 " type INT4 NOT NULL DEFAULT 0," " prio INT4 NOT NULL DEFAULT 0,"
299 " anonLevel INT4 NOT NULL DEFAULT 0,"
300 " expire INT8 NOT NULL DEFAULT 0," " rvalue INT8 NOT NULL,"
301 " hash TEXT NOT NULL DEFAULT ''," " vhash TEXT NOT NULL DEFAULT '',"
302 " value BLOB NOT NULL DEFAULT '')", NULL, NULL, NULL) != SQLITE_OK))
304 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
305 sqlite3_finalize (stmt);
306 return GNUNET_SYSERR;
308 sqlite3_finalize (stmt);
309 create_indices (plugin->dbh);
314 "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
315 &plugin->updPrio) != SQLITE_OK) ||
318 "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
319 &plugin->updRepl) != SQLITE_OK) ||
322 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
323 #if SQLITE_VERSION_NUMBER >= 3007000
324 "INDEXED BY idx_repl_rvalue "
326 "WHERE repl=?2 AND " " (rvalue>=?1 OR "
327 " NOT EXISTS (SELECT 1 FROM gn090 "
328 #if SQLITE_VERSION_NUMBER >= 3007000
329 "INDEXED BY idx_repl_rvalue "
331 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
332 "ORDER BY rvalue ASC LIMIT 1", &plugin->selRepl) != SQLITE_OK) ||
333 (sq_prepare (plugin->dbh, "SELECT MAX(repl) FROM gn090"
334 #if SQLITE_VERSION_NUMBER >= 3007000
335 " INDEXED BY idx_repl_rvalue"
337 "", &plugin->maxRepl) != SQLITE_OK) ||
340 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
341 #if SQLITE_VERSION_NUMBER >= 3007000
342 "INDEXED BY idx_expire "
344 "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
345 "ORDER BY expire ASC LIMIT 1", &plugin->selExpi) != SQLITE_OK) ||
348 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
349 #if SQLITE_VERSION_NUMBER >= 3007000
350 "INDEXED BY idx_anon_type_hash "
352 "WHERE (anonLevel = 0 AND type=?1) "
353 "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
354 &plugin->selZeroAnon) != SQLITE_OK) ||
357 "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
358 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
359 &plugin->insertContent) != SQLITE_OK) ||
361 (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?",
362 &plugin->delRow) != SQLITE_OK))
364 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "precompiling");
365 return GNUNET_SYSERR;
373 * Shutdown database connection and associate data
375 * @param plugin the plugin context (state for this module)
378 database_shutdown (struct Plugin *plugin)
382 #if SQLITE_VERSION_NUMBER >= 3007000
386 if (plugin->delRow != NULL)
387 sqlite3_finalize (plugin->delRow);
388 if (plugin->updPrio != NULL)
389 sqlite3_finalize (plugin->updPrio);
390 if (plugin->updRepl != NULL)
391 sqlite3_finalize (plugin->updRepl);
392 if (plugin->selRepl != NULL)
393 sqlite3_finalize (plugin->selRepl);
394 if (plugin->maxRepl != NULL)
395 sqlite3_finalize (plugin->maxRepl);
396 if (plugin->selExpi != NULL)
397 sqlite3_finalize (plugin->selExpi);
398 if (plugin->selZeroAnon != NULL)
399 sqlite3_finalize (plugin->selZeroAnon);
400 if (plugin->insertContent != NULL)
401 sqlite3_finalize (plugin->insertContent);
402 result = sqlite3_close (plugin->dbh);
403 #if SQLITE_VERSION_NUMBER >= 3007000
404 if (result == SQLITE_BUSY)
406 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
408 ("Tried to close sqlite without finalizing all prepared statements.\n"));
409 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
412 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
413 "Closing statement %p\n", stmt);
414 result = sqlite3_finalize (stmt);
415 if (result != SQLITE_OK)
416 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
417 "Failed to close statement %p: %d\n", stmt, result);
418 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
420 result = sqlite3_close (plugin->dbh);
423 if (SQLITE_OK != result)
424 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
426 GNUNET_free_non_null (plugin->fn);
431 * Delete the database entry with the given
434 * @param plugin the plugin context (state for this module)
435 * @param rid the ID of the row to delete
438 delete_by_rowid (struct Plugin *plugin, unsigned long long rid)
440 if (SQLITE_OK != sqlite3_bind_int64 (plugin->delRow, 1, rid))
442 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
443 "sqlite3_bind_XXXX");
444 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
445 LOG_SQLITE (plugin, NULL,
446 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
448 return GNUNET_SYSERR;
450 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
452 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
454 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
455 LOG_SQLITE (plugin, NULL,
456 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
458 return GNUNET_SYSERR;
460 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
461 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
468 * Store an item in the datastore.
471 * @param key key for the item
472 * @param size number of bytes in data
473 * @param data content stored
474 * @param type type of the content
475 * @param priority priority of the content
476 * @param anonymity anonymity-level for the content
477 * @param replication replication-level for the content
478 * @param expiration expiration time for the content
479 * @param msg set to an error message
480 * @return GNUNET_OK on success
483 sqlite_plugin_put (void *cls, const GNUNET_HashCode * key, uint32_t size,
484 const void *data, enum GNUNET_BLOCK_Type type,
485 uint32_t priority, uint32_t anonymity, uint32_t replication,
486 struct GNUNET_TIME_Absolute expiration, char **msg)
488 struct Plugin *plugin = cls;
492 GNUNET_HashCode vhash;
495 if (size > MAX_ITEM_SIZE)
496 return GNUNET_SYSERR;
497 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
498 "Storing in database block with type %u/key `%s'/priority %u/expiration in %llu ms (%lld).\n",
499 type, GNUNET_h2s (key), priority,
501 GNUNET_TIME_absolute_get_remaining (expiration).rel_value,
502 (long long) expiration.abs_value);
503 GNUNET_CRYPTO_hash (data, size, &vhash);
504 stmt = plugin->insertContent;
505 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
506 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) ||
507 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
508 (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
509 (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
510 (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value)) ||
511 (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) ||
513 sqlite3_bind_blob (stmt, 7, key, sizeof (GNUNET_HashCode),
514 SQLITE_TRANSIENT)) ||
516 sqlite3_bind_blob (stmt, 8, &vhash, sizeof (GNUNET_HashCode),
517 SQLITE_TRANSIENT)) ||
518 (SQLITE_OK != sqlite3_bind_blob (stmt, 9, data, size, SQLITE_TRANSIENT)))
520 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
521 "sqlite3_bind_XXXX");
522 if (SQLITE_OK != sqlite3_reset (stmt))
523 LOG_SQLITE (plugin, NULL,
524 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
526 return GNUNET_SYSERR;
528 n = sqlite3_step (stmt);
532 plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
533 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
534 "Stored new entry (%u bytes)\n",
535 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
540 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
545 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
547 if (SQLITE_OK != sqlite3_reset (stmt))
548 LOG_SQLITE (plugin, NULL,
549 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
551 database_shutdown (plugin);
552 database_setup (plugin->env->cfg, plugin);
553 return GNUNET_SYSERR;
555 if (SQLITE_OK != sqlite3_reset (stmt))
556 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
563 * Update the priority for a particular key in the datastore. If
564 * the expiration time in value is different than the time found in
565 * the datastore, the higher value should be kept. For the
566 * anonymity level, the lower value is to be used. The specified
567 * priority should be added to the existing priority, ignoring the
570 * Note that it is possible for multiple values to match this put.
571 * In that case, all of the respective values are updated.
573 * @param cls the plugin context (state for this module)
574 * @param uid unique identifier of the datum
575 * @param delta by how much should the priority
576 * change? If priority + delta < 0 the
577 * priority should be set to 0 (never go
579 * @param expire new expiration time should be the
580 * MAX of any existing expiration time and
582 * @param msg set to an error message
583 * @return GNUNET_OK on success
586 sqlite_plugin_update (void *cls, uint64_t uid, int delta,
587 struct GNUNET_TIME_Absolute expire, char **msg)
589 struct Plugin *plugin = cls;
592 if ((SQLITE_OK != sqlite3_bind_int (plugin->updPrio, 1, delta)) ||
593 (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value))
594 || (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 3, uid)))
596 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
597 "sqlite3_bind_XXXX");
598 if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
599 LOG_SQLITE (plugin, NULL,
600 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
602 return GNUNET_SYSERR;
605 n = sqlite3_step (plugin->updPrio);
606 if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
607 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
612 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Block updated\n");
615 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
619 LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
621 return GNUNET_SYSERR;
627 * Execute statement that gets a row and call the callback
628 * with the result. Resets the statement afterwards.
630 * @param plugin the plugin
631 * @param stmt the statement
632 * @param proc processor to call
633 * @param proc_cls closure for 'proc'
636 execute_get (struct Plugin *plugin, sqlite3_stmt * stmt,
637 PluginDatumProcessor proc, void *proc_cls)
640 struct GNUNET_TIME_Absolute expiration;
641 unsigned long long rowid;
645 n = sqlite3_step (stmt);
649 size = sqlite3_column_bytes (stmt, 5);
650 rowid = sqlite3_column_int64 (stmt, 6);
651 if (sqlite3_column_bytes (stmt, 4) != sizeof (GNUNET_HashCode))
653 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
655 ("Invalid data in database. Trying to fix (by deletion).\n"));
656 if (SQLITE_OK != sqlite3_reset (stmt))
657 LOG_SQLITE (plugin, NULL,
658 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
660 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
661 plugin->env->duc (plugin->env->cls,
662 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
665 expiration.abs_value = sqlite3_column_int64 (stmt, 3);
666 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
667 "Found reply in database with expiration %llu\n",
668 (unsigned long long) expiration.abs_value);
669 ret = proc (proc_cls, sqlite3_column_blob (stmt, 4) /* key */ ,
670 size, sqlite3_column_blob (stmt, 5) /* data */ ,
671 sqlite3_column_int (stmt, 0) /* type */ ,
672 sqlite3_column_int (stmt, 1) /* priority */ ,
673 sqlite3_column_int (stmt, 2) /* anonymity */ ,
675 if (SQLITE_OK != sqlite3_reset (stmt))
676 LOG_SQLITE (plugin, NULL,
677 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
679 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)))
680 plugin->env->duc (plugin->env->cls,
681 -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
684 /* database must be empty */
685 if (SQLITE_OK != sqlite3_reset (stmt))
686 LOG_SQLITE (plugin, NULL,
687 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
694 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
696 if (SQLITE_OK != sqlite3_reset (stmt))
697 LOG_SQLITE (plugin, NULL,
698 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
701 database_shutdown (plugin);
702 database_setup (plugin->env->cfg, plugin);
705 if (SQLITE_OK != sqlite3_reset (stmt))
706 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
708 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
714 * Select a subset of the items in the datastore and call
715 * the given processor for the item.
717 * @param cls our plugin context
718 * @param offset offset of the result (modulo num-results);
719 * specific ordering does not matter for the offset
720 * @param type entries of which type should be considered?
721 * Use 0 for any type.
722 * @param proc function to call on each matching value;
723 * will be called once with a NULL value at the end
724 * @param proc_cls closure for proc
727 sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
728 enum GNUNET_BLOCK_Type type,
729 PluginDatumProcessor proc, void *proc_cls)
731 struct Plugin *plugin = cls;
734 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
735 stmt = plugin->selZeroAnon;
736 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
737 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)))
739 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
740 "sqlite3_bind_XXXX");
741 if (SQLITE_OK != sqlite3_reset (stmt))
742 LOG_SQLITE (plugin, NULL,
743 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
745 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
748 execute_get (plugin, stmt, proc, proc_cls);
754 * Get results for a particular key in the datastore.
757 * @param offset offset (mod count).
758 * @param key key to match, never NULL
759 * @param vhash hash of the value, maybe NULL (to
760 * match all values that have the right key).
761 * Note that for DBlocks there is no difference
762 * betwen key and vhash, but for other blocks
764 * @param type entries of which type are relevant?
765 * Use 0 for any type.
766 * @param proc function to call on each matching value;
767 * will be called once with a NULL value at the end
768 * @param proc_cls closure for proc
771 sqlite_plugin_get_key (void *cls, uint64_t offset, const GNUNET_HashCode * key,
772 const GNUNET_HashCode * vhash,
773 enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc,
776 struct Plugin *plugin = cls;
784 GNUNET_assert (proc != NULL);
785 GNUNET_assert (key != NULL);
786 GNUNET_snprintf (scratch, sizeof (scratch),
787 "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
788 vhash == NULL ? "" : " AND vhash=?",
789 type == 0 ? "" : " AND type=?");
790 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
792 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
794 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
799 sqlite3_bind_blob (stmt, sqoff++, key, sizeof (GNUNET_HashCode),
801 if ((vhash != NULL) && (ret == SQLITE_OK))
803 sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (GNUNET_HashCode),
805 if ((type != 0) && (ret == SQLITE_OK))
806 ret = sqlite3_bind_int (stmt, sqoff++, type);
807 if (SQLITE_OK != ret)
809 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
810 sqlite3_finalize (stmt);
811 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
814 ret = sqlite3_step (stmt);
815 if (ret != SQLITE_ROW)
817 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
819 sqlite3_finalize (stmt);
820 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
823 total = sqlite3_column_int (stmt, 0);
824 sqlite3_finalize (stmt);
827 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
830 limit_off = (int) (offset % total);
833 GNUNET_snprintf (scratch, sizeof (scratch),
834 "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
835 "FROM gn090 WHERE hash=?%s%s "
836 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
837 vhash == NULL ? "" : " AND vhash=?",
838 type == 0 ? "" : " AND type=?");
839 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
841 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
843 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
848 sqlite3_bind_blob (stmt, sqoff++, key, sizeof (GNUNET_HashCode),
850 if ((vhash != NULL) && (ret == SQLITE_OK))
852 sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (GNUNET_HashCode),
854 if ((type != 0) && (ret == SQLITE_OK))
855 ret = sqlite3_bind_int (stmt, sqoff++, type);
856 if (ret == SQLITE_OK)
857 ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
858 if (ret != SQLITE_OK)
860 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
862 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
865 execute_get (plugin, stmt, proc, proc_cls);
866 sqlite3_finalize (stmt);
872 * Context for 'repl_proc' function.
878 * Function to call for the result (or the NULL).
880 PluginDatumProcessor proc;
893 * Yes if UID was set.
900 * Wrapper for the processor for 'sqlite_plugin_replication_get'.
901 * Decrements the replication counter and calls the original
905 * @param key key for the content
906 * @param size number of bytes in data
907 * @param data content stored
908 * @param type type of the content
909 * @param priority priority of the content
910 * @param anonymity anonymity-level for the content
911 * @param expiration expiration time for the content
912 * @param uid unique identifier for the datum;
913 * maybe 0 if no unique identifier is available
915 * @return GNUNET_OK for normal return,
916 * GNUNET_NO to delete the item
919 repl_proc (void *cls, const GNUNET_HashCode * key, uint32_t size,
920 const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority,
921 uint32_t anonymity, struct GNUNET_TIME_Absolute expiration,
924 struct ReplCtx *rc = cls;
928 rc->proc (rc->proc_cls, key, size, data, type, priority, anonymity,
933 rc->have_uid = GNUNET_YES;
940 * Get a random item for replication. Returns a single random item
941 * from those with the highest replication counters. The item's
942 * replication counter is decremented by one IF it was positive before.
943 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
946 * @param proc function to call the value (once only).
947 * @param proc_cls closure for proc
950 sqlite_plugin_get_replication (void *cls, PluginDatumProcessor proc,
953 struct Plugin *plugin = cls;
959 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
960 "Getting random block based on replication order.\n");
961 rc.have_uid = GNUNET_NO;
963 rc.proc_cls = proc_cls;
964 stmt = plugin->maxRepl;
965 if (SQLITE_ROW != sqlite3_step (stmt))
967 if (SQLITE_OK != sqlite3_reset (stmt))
968 LOG_SQLITE (plugin, NULL,
969 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
972 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
975 repl = sqlite3_column_int (stmt, 0);
976 if (SQLITE_OK != sqlite3_reset (stmt))
977 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
979 stmt = plugin->selRepl;
980 rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
981 if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue))
983 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
984 "sqlite3_bind_XXXX");
985 if (SQLITE_OK != sqlite3_reset (stmt))
986 LOG_SQLITE (plugin, NULL,
987 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
989 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
992 if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl))
994 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
995 "sqlite3_bind_XXXX");
996 if (SQLITE_OK != sqlite3_reset (stmt))
997 LOG_SQLITE (plugin, NULL,
998 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1000 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1003 execute_get (plugin, stmt, &repl_proc, &rc);
1004 if (GNUNET_YES == rc.have_uid)
1006 if (SQLITE_OK != sqlite3_bind_int64 (plugin->updRepl, 1, rc.uid))
1008 LOG_SQLITE (plugin, NULL,
1009 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1010 "sqlite3_bind_XXXX");
1011 if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
1012 LOG_SQLITE (plugin, NULL,
1013 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1017 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1018 LOG_SQLITE (plugin, NULL,
1019 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1021 if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
1022 LOG_SQLITE (plugin, NULL,
1023 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1031 * Get a random item that has expired or has low priority.
1032 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1034 * @param cls closure
1035 * @param proc function to call the value (once only).
1036 * @param proc_cls closure for proc
1039 sqlite_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
1042 struct Plugin *plugin = cls;
1044 struct GNUNET_TIME_Absolute now;
1046 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1047 "Getting random block based on expiration and priority order.\n");
1048 now = GNUNET_TIME_absolute_get ();
1049 stmt = plugin->selExpi;
1050 if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value))
1052 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1053 "sqlite3_bind_XXXX");
1054 if (SQLITE_OK != sqlite3_reset (stmt))
1055 LOG_SQLITE (plugin, NULL,
1056 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1058 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1061 execute_get (plugin, stmt, proc, proc_cls);
1067 * Get all of the keys in the datastore.
1069 * @param cls closure
1070 * @param proc function to call on each key
1071 * @param proc_cls closure for proc
1074 sqlite_plugin_get_keys (void *cls,
1075 PluginKeyProcessor proc,
1078 struct Plugin *plugin = cls;
1079 const GNUNET_HashCode *key;
1083 GNUNET_assert (proc != NULL);
1084 if (sq_prepare (plugin->dbh, "SELECT hash FROM gn090", &stmt) != SQLITE_OK)
1086 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
1090 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1092 key = sqlite3_column_blob (stmt, 1);
1093 if (sizeof (GNUNET_HashCode) == sqlite3_column_bytes (stmt, 1))
1094 proc (proc_cls, key, 1);
1096 if (SQLITE_DONE != ret)
1097 LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1098 sqlite3_finalize (stmt);
1105 * @param cls our plugin context
1108 sqlite_plugin_drop (void *cls)
1110 struct Plugin *plugin = cls;
1112 plugin->drop_on_shutdown = GNUNET_YES;
1117 * Get an estimate of how much space the database is
1120 * @param cls the 'struct Plugin'
1121 * @return the size of the database on disk (estimate)
1123 static unsigned long long
1124 sqlite_plugin_estimate_size (void *cls)
1126 struct Plugin *plugin = cls;
1135 if (SQLITE_VERSION_NUMBER < 3006000)
1137 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "datastore-sqlite",
1139 ("sqlite version to old to determine size, assuming zero\n"));
1142 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1144 sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
1146 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt));
1147 if (SQLITE_ROW == sqlite3_step (stmt))
1148 pages = sqlite3_column_int64 (stmt, 0);
1151 sqlite3_finalize (stmt);
1152 CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt));
1153 CHECK (SQLITE_ROW == sqlite3_step (stmt));
1154 page_size = sqlite3_column_int64 (stmt, 0);
1155 sqlite3_finalize (stmt);
1156 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1158 ("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1159 (unsigned long long) pages, (unsigned long long) page_size);
1160 return pages * page_size;
1165 * Entry point for the plugin.
1167 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
1168 * @return NULL on error, othrewise the plugin context
1171 libgnunet_plugin_datastore_sqlite_init (void *cls)
1173 static struct Plugin plugin;
1174 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1175 struct GNUNET_DATASTORE_PluginFunctions *api;
1177 if (plugin.env != NULL)
1178 return NULL; /* can only initialize once! */
1179 memset (&plugin, 0, sizeof (struct Plugin));
1181 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1183 database_shutdown (&plugin);
1186 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1188 api->estimate_size = &sqlite_plugin_estimate_size;
1189 api->put = &sqlite_plugin_put;
1190 api->update = &sqlite_plugin_update;
1191 api->get_key = &sqlite_plugin_get_key;
1192 api->get_replication = &sqlite_plugin_get_replication;
1193 api->get_expiration = &sqlite_plugin_get_expiration;
1194 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1195 api->get_keys = &sqlite_plugin_get_keys;
1196 api->drop = &sqlite_plugin_drop;
1197 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite",
1198 _("Sqlite database running\n"));
1204 * Exit point from the plugin.
1206 * @param cls the plugin context (as returned by "init")
1207 * @return always NULL
1210 libgnunet_plugin_datastore_sqlite_done (void *cls)
1213 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1214 struct Plugin *plugin = api->cls;
1216 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1217 "sqlite plugin is done\n");
1219 if (plugin->drop_on_shutdown)
1220 fn = GNUNET_strdup (plugin->fn);
1221 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1222 "Shutting down database\n");
1223 database_shutdown (plugin);
1228 if (0 != UNLINK (fn))
1229 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
1232 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
1233 "sqlite plugin is finished\n");
1237 /* end of plugin_datastore_sqlite.c */