2 This file is part of GNUnet
3 (C) 2009, 2011 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file datastore/plugin_datastore_sqlite.c
23 * @brief sqlite-based datastore backend
24 * @author Christian Grothoff
28 #include "gnunet_datastore_plugin.h"
32 * Enable or disable logging debug messages.
34 #define DEBUG_SQLITE GNUNET_NO
37 * We allocate items on the stack at times. To prevent a stack
38 * overflow, we impose a limit on the maximum size for the data per
39 * item. 64k should be enough.
41 #define MAX_ITEM_SIZE 65536
44 * After how many ms "busy" should a DB operation fail for good?
45 * A low value makes sure that we are more responsive to requests
46 * (especially PUTs). A high value guarantees a higher success
47 * rate (SELECTs in iterate can take several seconds despite LIMIT=1).
49 * The default value of 250ms should ensure that users do not experience
50 * huge latencies while at the same time allowing operations to succeed
51 * with reasonable probability.
53 #define BUSY_TIMEOUT_MS 250
57 * Log an error message at log-level 'level' that indicates
58 * a failure of the command 'cmd' on file 'filename'
59 * with the message given by strerror(errno).
61 #define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed at %s:%u with error: %s"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); } while(0)
66 * Context for all functions in this plugin.
71 * Our execution environment.
73 struct GNUNET_DATASTORE_PluginEnvironment *env;
81 * Native SQLite database handle.
86 * Precompiled SQL for deletion.
91 * Precompiled SQL for update.
93 sqlite3_stmt *updPrio;
96 * Precompiled SQL for replication decrement.
98 sqlite3_stmt *updRepl;
101 * Precompiled SQL for replication selection.
103 sqlite3_stmt *selRepl;
106 * Precompiled SQL for expiration selection.
108 sqlite3_stmt *selExpi;
111 * Precompiled SQL for expiration selection.
113 sqlite3_stmt *selZeroAnon;
116 * Precompiled SQL for insertion.
118 sqlite3_stmt *insertContent;
121 * Should the database be dropped on shutdown?
123 int drop_on_shutdown;
129 * @brief Prepare a SQL statement
131 * @param dbh handle to the database
132 * @param zSql SQL statement, UTF-8 encoded
133 * @param ppStmt set to the prepared statement
134 * @return 0 on success
137 sq_prepare (sqlite3 * dbh,
139 sqlite3_stmt ** ppStmt)
144 result = sqlite3_prepare_v2 (dbh,
148 (const char **) &dummy);
149 #if DEBUG_SQLITE && 0
150 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
152 "Prepared `%s' / %p: %d\n",
162 * Create our database indices.
164 * @param dbh handle to the database
167 create_indices (sqlite3 * dbh)
171 "CREATE INDEX idx_hash ON gn090 (hash)", NULL, NULL, NULL);
173 "CREATE INDEX idx_hash_vhash ON gn090 (hash,vhash)", NULL,
175 sqlite3_exec (dbh, "CREATE INDEX idx_expire_repl ON gn090 (expire ASC,repl DESC)", NULL, NULL,
177 sqlite3_exec (dbh, "CREATE INDEX idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)",
179 sqlite3_exec (dbh, "CREATE INDEX idx_expire ON gn090 (expire)",
181 sqlite3_exec (dbh, "CREATE INDEX idx_repl ON gn090 (repl)",
187 #define CHECK(a) GNUNET_break(a)
191 #define ENULL_DEFINED 1
192 #define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "%s\n", e); sqlite3_free(e); }
197 * Initialize the database connections and associated
198 * data structures (create tables and indices
199 * as needed as well).
201 * @param cfg our configuration
202 * @param plugin the plugin context (state for this module)
203 * @return GNUNET_OK on success
206 database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
207 struct Plugin *plugin)
216 GNUNET_CONFIGURATION_get_value_filename (cfg,
221 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
223 _("Option `%s' in section `%s' missing in configuration!\n"),
226 return GNUNET_SYSERR;
228 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
230 if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
233 GNUNET_free (afsdir);
234 return GNUNET_SYSERR;
236 /* database is new or got deleted, reset payload to zero! */
237 plugin->env->duc (plugin->env->cls, 0);
240 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
241 nl_langinfo (CODESET));
243 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
244 "UTF-8"); /* good luck */
246 GNUNET_free (afsdir);
248 /* Open database and precompile statements */
249 if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK)
251 GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
253 _("Unable to initialize SQLite: %s.\n"),
254 sqlite3_errmsg (plugin->dbh));
255 return GNUNET_SYSERR;
258 sqlite3_exec (plugin->dbh,
259 "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
261 sqlite3_exec (plugin->dbh,
262 "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
264 sqlite3_exec (plugin->dbh,
265 "PRAGMA legacy_file_format=OFF", NULL, NULL, ENULL));
267 sqlite3_exec (plugin->dbh,
268 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
270 sqlite3_exec (plugin->dbh,
271 "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL, ENULL));
273 sqlite3_exec (plugin->dbh,
274 "PRAGMA count_changes=OFF", NULL, NULL, ENULL));
276 sqlite3_exec (plugin->dbh,
277 "PRAGMA page_size=4092", NULL, NULL, ENULL));
279 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
282 /* We have to do it here, because otherwise precompiling SQL might fail */
284 sq_prepare (plugin->dbh,
285 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn090'",
287 if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
288 (sqlite3_exec (plugin->dbh,
289 "CREATE TABLE gn090 ("
290 " repl INT4 NOT NULL DEFAULT 0,"
291 " type INT4 NOT NULL DEFAULT 0,"
292 " prio INT4 NOT NULL DEFAULT 0,"
293 " anonLevel INT4 NOT NULL DEFAULT 0,"
294 " expire INT8 NOT NULL DEFAULT 0,"
295 " hash TEXT NOT NULL DEFAULT '',"
296 " vhash TEXT NOT NULL DEFAULT '',"
297 " value BLOB NOT NULL DEFAULT '')", NULL, NULL,
298 NULL) != SQLITE_OK) )
300 LOG_SQLITE (plugin, NULL,
301 GNUNET_ERROR_TYPE_ERROR,
303 sqlite3_finalize (stmt);
304 return GNUNET_SYSERR;
306 sqlite3_finalize (stmt);
307 create_indices (plugin->dbh);
309 if ((sq_prepare (plugin->dbh,
310 "UPDATE gn090 SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
311 &plugin->updPrio) != SQLITE_OK) ||
312 (sq_prepare (plugin->dbh,
313 "UPDATE gn090 SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
314 &plugin->updRepl) != SQLITE_OK) ||
315 (sq_prepare (plugin->dbh,
316 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090"
317 " ORDER BY repl DESC, Random() LIMIT 1",
318 &plugin->selRepl) != SQLITE_OK) ||
319 (sq_prepare (plugin->dbh,
320 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 "
321 " WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR expire < ?1 "
322 " ORDER BY prio ASC LIMIT 1",
323 &plugin->selExpi) != SQLITE_OK) ||
324 (sq_prepare (plugin->dbh,
325 "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ FROM gn090 "
326 "WHERE (anonLevel = 0 AND type=?1) "
327 "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
328 &plugin->selZeroAnon) != SQLITE_OK) ||
329 (sq_prepare (plugin->dbh,
330 "INSERT INTO gn090 (repl, type, prio, "
331 "anonLevel, expire, hash, vhash, value) "
332 "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
333 &plugin->insertContent) != SQLITE_OK) ||
334 (sq_prepare (plugin->dbh,
335 "DELETE FROM gn090 WHERE _ROWID_ = ?",
336 &plugin->delRow) != SQLITE_OK))
338 LOG_SQLITE (plugin, NULL,
339 GNUNET_ERROR_TYPE_ERROR, "precompiling");
340 return GNUNET_SYSERR;
348 * Shutdown database connection and associate data
350 * @param plugin the plugin context (state for this module)
353 database_shutdown (struct Plugin *plugin)
356 #if SQLITE_VERSION_NUMBER >= 3007000
360 if (plugin->delRow != NULL)
361 sqlite3_finalize (plugin->delRow);
362 if (plugin->updPrio != NULL)
363 sqlite3_finalize (plugin->updPrio);
364 if (plugin->updRepl != NULL)
365 sqlite3_finalize (plugin->updRepl);
366 if (plugin->selRepl != NULL)
367 sqlite3_finalize (plugin->selRepl);
368 if (plugin->selExpi != NULL)
369 sqlite3_finalize (plugin->selExpi);
370 if (plugin->selZeroAnon != NULL)
371 sqlite3_finalize (plugin->selZeroAnon);
372 if (plugin->insertContent != NULL)
373 sqlite3_finalize (plugin->insertContent);
374 result = sqlite3_close(plugin->dbh);
375 #if SQLITE_VERSION_NUMBER >= 3007000
376 if (result == SQLITE_BUSY)
378 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
380 _("Tried to close sqlite without finalizing all prepared statements.\n"));
381 stmt = sqlite3_next_stmt(plugin->dbh, NULL);
385 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
386 "sqlite", "Closing statement %p\n", stmt);
388 result = sqlite3_finalize(stmt);
390 if (result != SQLITE_OK)
391 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
393 "Failed to close statement %p: %d\n", stmt, result);
395 stmt = sqlite3_next_stmt(plugin->dbh, NULL);
397 result = sqlite3_close(plugin->dbh);
400 if (SQLITE_OK != result)
401 LOG_SQLITE (plugin, NULL,
402 GNUNET_ERROR_TYPE_ERROR,
405 GNUNET_free_non_null (plugin->fn);
410 * Delete the database entry with the given
413 * @param plugin the plugin context (state for this module)
414 * @param rid the ID of the row to delete
417 delete_by_rowid (struct Plugin* plugin,
418 unsigned long long rid)
420 sqlite3_bind_int64 (plugin->delRow, 1, rid);
421 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
423 LOG_SQLITE (plugin, NULL,
424 GNUNET_ERROR_TYPE_ERROR |
425 GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
426 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
427 LOG_SQLITE (plugin, NULL,
428 GNUNET_ERROR_TYPE_ERROR |
429 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
430 return GNUNET_SYSERR;
432 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
433 LOG_SQLITE (plugin, NULL,
434 GNUNET_ERROR_TYPE_ERROR |
435 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
441 * Store an item in the datastore.
444 * @param key key for the item
445 * @param size number of bytes in data
446 * @param data content stored
447 * @param type type of the content
448 * @param priority priority of the content
449 * @param anonymity anonymity-level for the content
450 * @param replication replication-level for the content
451 * @param expiration expiration time for the content
452 * @param msg set to an error message
453 * @return GNUNET_OK on success
456 sqlite_plugin_put (void *cls,
457 const GNUNET_HashCode *key,
460 enum GNUNET_BLOCK_Type type,
463 uint32_t replication,
464 struct GNUNET_TIME_Absolute expiration,
467 struct Plugin *plugin = cls;
471 GNUNET_HashCode vhash;
473 if (size > MAX_ITEM_SIZE)
474 return GNUNET_SYSERR;
476 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
478 "Storing in database block with type %u/key `%s'/priority %u/expiration in %llu ms (%lld).\n",
482 (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).rel_value,
483 (long long) expiration.abs_value);
485 GNUNET_CRYPTO_hash (data, size, &vhash);
486 stmt = plugin->insertContent;
487 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) ||
488 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
489 (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
490 (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
491 (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value)) ||
493 sqlite3_bind_blob (stmt, 6, key, sizeof (GNUNET_HashCode),
494 SQLITE_TRANSIENT)) ||
496 sqlite3_bind_blob (stmt, 7, &vhash, sizeof (GNUNET_HashCode),
499 sqlite3_bind_blob (stmt, 8, data, size,
504 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX");
505 if (SQLITE_OK != sqlite3_reset (stmt))
506 LOG_SQLITE (plugin, NULL,
507 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
508 return GNUNET_SYSERR;
510 n = sqlite3_step (stmt);
514 plugin->env->duc (plugin->env->cls,
515 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
517 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
519 "Stored new entry (%u bytes)\n",
520 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
526 LOG_SQLITE (plugin, msg,
527 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
532 LOG_SQLITE (plugin, msg,
533 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
535 if (SQLITE_OK != sqlite3_reset (stmt))
536 LOG_SQLITE (plugin, NULL,
537 GNUNET_ERROR_TYPE_ERROR |
538 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
539 database_shutdown (plugin);
540 database_setup (plugin->env->cfg,
542 return GNUNET_SYSERR;
544 if (SQLITE_OK != sqlite3_reset (stmt))
545 LOG_SQLITE (plugin, NULL,
546 GNUNET_ERROR_TYPE_ERROR |
547 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
553 * Update the priority for a particular key in the datastore. If
554 * the expiration time in value is different than the time found in
555 * the datastore, the higher value should be kept. For the
556 * anonymity level, the lower value is to be used. The specified
557 * priority should be added to the existing priority, ignoring the
560 * Note that it is possible for multiple values to match this put.
561 * In that case, all of the respective values are updated.
563 * @param cls the plugin context (state for this module)
564 * @param uid unique identifier of the datum
565 * @param delta by how much should the priority
566 * change? If priority + delta < 0 the
567 * priority should be set to 0 (never go
569 * @param expire new expiration time should be the
570 * MAX of any existing expiration time and
572 * @param msg set to an error message
573 * @return GNUNET_OK on success
576 sqlite_plugin_update (void *cls,
578 int delta, struct GNUNET_TIME_Absolute expire,
581 struct Plugin *plugin = cls;
584 sqlite3_bind_int (plugin->updPrio, 1, delta);
585 sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value);
586 sqlite3_bind_int64 (plugin->updPrio, 3, uid);
587 n = sqlite3_step (plugin->updPrio);
588 sqlite3_reset (plugin->updPrio);
593 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
599 LOG_SQLITE (plugin, msg,
600 GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
604 LOG_SQLITE (plugin, msg,
605 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
607 return GNUNET_SYSERR;
613 * Execute statement that gets a row and call the callback
614 * with the result. Resets the statement afterwards.
616 * @param plugin the plugin
617 * @param stmt the statement
618 * @param proc processor to call
619 * @param proc_cls closure for 'proc'
622 execute_get (struct Plugin *plugin,
624 PluginDatumProcessor proc, void *proc_cls)
627 struct GNUNET_TIME_Absolute expiration;
628 unsigned long long rowid;
632 n = sqlite3_step (stmt);
636 size = sqlite3_column_bytes (stmt, 5);
637 rowid = sqlite3_column_int64 (stmt, 6);
638 if (sqlite3_column_bytes (stmt, 4) != sizeof (GNUNET_HashCode))
640 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
642 _("Invalid data in database. Trying to fix (by deletion).\n"));
643 if (SQLITE_OK != sqlite3_reset (stmt))
644 LOG_SQLITE (plugin, NULL,
645 GNUNET_ERROR_TYPE_ERROR |
646 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
647 if (GNUNET_OK == delete_by_rowid (plugin, rowid))
648 plugin->env->duc (plugin->env->cls,
649 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
652 expiration.abs_value = sqlite3_column_int64 (stmt, 3);
654 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
656 "Found reply in database with expiration %llu\n",
657 (unsigned long long) expiration.abs_value);
659 ret = proc (proc_cls,
660 sqlite3_column_blob (stmt, 4) /* key */,
662 sqlite3_column_blob (stmt, 5) /* data */,
663 sqlite3_column_int (stmt, 0) /* type */,
664 sqlite3_column_int (stmt, 1) /* priority */,
665 sqlite3_column_int (stmt, 2) /* anonymity */,
668 if (SQLITE_OK != sqlite3_reset (stmt))
669 LOG_SQLITE (plugin, NULL,
670 GNUNET_ERROR_TYPE_ERROR |
671 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
672 if ( (GNUNET_NO == ret) &&
673 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
674 plugin->env->duc (plugin->env->cls,
675 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
678 /* database must be empty */
679 if (SQLITE_OK != sqlite3_reset (stmt))
680 LOG_SQLITE (plugin, NULL,
681 GNUNET_ERROR_TYPE_ERROR |
682 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
688 LOG_SQLITE (plugin, NULL,
689 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
691 if (SQLITE_OK != sqlite3_reset (stmt))
692 LOG_SQLITE (plugin, NULL,
693 GNUNET_ERROR_TYPE_ERROR |
694 GNUNET_ERROR_TYPE_BULK,
697 database_shutdown (plugin);
698 database_setup (plugin->env->cfg,
702 if (SQLITE_OK != sqlite3_reset (stmt))
703 LOG_SQLITE (plugin, NULL,
704 GNUNET_ERROR_TYPE_ERROR |
705 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
706 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
707 GNUNET_TIME_UNIT_ZERO_ABS, 0);
713 * Select a subset of the items in the datastore and call
714 * the given processor for the item.
716 * @param cls our plugin context
717 * @param type entries of which type should be considered?
718 * Use 0 for any type.
719 * @param proc function to call on each matching value;
720 * will be called once with a NULL value at the end
721 * @param proc_cls closure for proc
724 sqlite_plugin_get_zero_anonymity (void *cls,
726 enum GNUNET_BLOCK_Type type,
727 PluginDatumProcessor proc,
730 struct Plugin *plugin = cls;
733 GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
734 stmt = plugin->selZeroAnon;
735 if ( (SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
736 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)) )
738 LOG_SQLITE (plugin, NULL,
739 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
740 "sqlite3_bind_XXXX");
741 if (SQLITE_OK != sqlite3_reset (stmt))
742 LOG_SQLITE (plugin, NULL,
743 GNUNET_ERROR_TYPE_ERROR |
744 GNUNET_ERROR_TYPE_BULK,
746 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
747 GNUNET_TIME_UNIT_ZERO_ABS, 0);
750 execute_get (plugin, stmt, proc, proc_cls);
756 * Get results for a particular key in the datastore.
759 * @param offset offset (mod count).
760 * @param key key to match, never NULL
761 * @param vhash hash of the value, maybe NULL (to
762 * match all values that have the right key).
763 * Note that for DBlocks there is no difference
764 * betwen key and vhash, but for other blocks
766 * @param type entries of which type are relevant?
767 * Use 0 for any type.
768 * @param proc function to call on each matching value;
769 * will be called once with a NULL value at the end
770 * @param proc_cls closure for proc
773 sqlite_plugin_get_key (void *cls,
775 const GNUNET_HashCode *key,
776 const GNUNET_HashCode *vhash,
777 enum GNUNET_BLOCK_Type type,
778 PluginDatumProcessor proc, void *proc_cls)
780 struct Plugin *plugin = cls;
788 GNUNET_assert (proc != NULL);
789 GNUNET_assert (key != NULL);
790 GNUNET_snprintf (scratch, sizeof (scratch),
791 "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
792 vhash == NULL ? "" : " AND vhash=?",
793 type == 0 ? "" : " AND type=?");
794 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
796 LOG_SQLITE (plugin, NULL,
797 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
798 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
802 ret = sqlite3_bind_blob (stmt, sqoff++,
803 key, sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
804 if ((vhash != NULL) && (ret == SQLITE_OK))
805 ret = sqlite3_bind_blob (stmt, sqoff++,
807 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
808 if ((type != 0) && (ret == SQLITE_OK))
809 ret = sqlite3_bind_int (stmt, sqoff++, type);
810 if (SQLITE_OK != ret)
812 LOG_SQLITE (plugin, NULL,
813 GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
814 sqlite3_finalize (stmt);
815 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
818 ret = sqlite3_step (stmt);
819 if (ret != SQLITE_ROW)
821 LOG_SQLITE (plugin, NULL,
822 GNUNET_ERROR_TYPE_ERROR| GNUNET_ERROR_TYPE_BULK,
824 sqlite3_finalize (stmt);
825 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
828 total = sqlite3_column_int (stmt, 0);
829 sqlite3_finalize (stmt);
832 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
835 limit_off = (int) (offset % total);
838 GNUNET_snprintf (scratch, sizeof (scratch),
839 "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
840 "FROM gn090 WHERE hash=?%s%s "
841 "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
842 vhash == NULL ? "" : " AND vhash=?",
843 type == 0 ? "" : " AND type=?");
844 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
846 LOG_SQLITE (plugin, NULL,
847 GNUNET_ERROR_TYPE_ERROR |
848 GNUNET_ERROR_TYPE_BULK, "sqlite_prepare");
849 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
853 ret = sqlite3_bind_blob (stmt,
856 sizeof (GNUNET_HashCode),
858 if ((vhash != NULL) && (ret == SQLITE_OK))
859 ret = sqlite3_bind_blob (stmt,
862 sizeof (GNUNET_HashCode), SQLITE_TRANSIENT);
863 if ((type != 0) && (ret == SQLITE_OK))
864 ret = sqlite3_bind_int (stmt, sqoff++, type);
865 if (ret == SQLITE_OK)
866 ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
867 if (ret != SQLITE_OK)
869 LOG_SQLITE (plugin, NULL,
870 GNUNET_ERROR_TYPE_ERROR |
871 GNUNET_ERROR_TYPE_BULK, "sqlite_bind");
872 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
875 execute_get (plugin, stmt, proc, proc_cls);
876 sqlite3_finalize (stmt);
882 * Context for 'repl_proc' function.
890 struct Plugin *plugin;
893 * Function to call for the result (or the NULL).
895 PluginDatumProcessor proc;
905 * Wrapper for the processor for 'sqlite_plugin_replication_get'.
906 * Decrements the replication counter and calls the original
910 * @param key key for the content
911 * @param size number of bytes in data
912 * @param data content stored
913 * @param type type of the content
914 * @param priority priority of the content
915 * @param anonymity anonymity-level for the content
916 * @param expiration expiration time for the content
917 * @param uid unique identifier for the datum;
918 * maybe 0 if no unique identifier is available
920 * @return GNUNET_OK for normal return,
921 * GNUNET_NO to delete the item
924 repl_proc (void *cls,
925 const GNUNET_HashCode *key,
928 enum GNUNET_BLOCK_Type type,
931 struct GNUNET_TIME_Absolute expiration,
934 struct ReplCtx *rc = cls;
935 struct Plugin *plugin = rc->plugin;
938 ret = rc->proc (rc->proc_cls,
941 type, priority, anonymity, expiration,
945 sqlite3_bind_int64 (plugin->updRepl, 1, uid);
946 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
948 LOG_SQLITE (plugin, NULL,
949 GNUNET_ERROR_TYPE_ERROR |
950 GNUNET_ERROR_TYPE_BULK, "sqlite3_step");
951 if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
952 LOG_SQLITE (plugin, NULL,
953 GNUNET_ERROR_TYPE_ERROR |
954 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
955 return GNUNET_SYSERR;
957 if (SQLITE_OK != sqlite3_reset (plugin->delRow))
958 LOG_SQLITE (plugin, NULL,
959 GNUNET_ERROR_TYPE_ERROR |
960 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
967 * Get a random item for replication. Returns a single random item
968 * from those with the highest replication counters. The item's
969 * replication counter is decremented by one IF it was positive before.
970 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
973 * @param proc function to call the value (once only).
974 * @param proc_cls closure for proc
977 sqlite_plugin_get_replication (void *cls,
978 PluginDatumProcessor proc, void *proc_cls)
980 struct Plugin *plugin = cls;
984 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
986 "Getting random block based on replication order.\n");
990 rc.proc_cls = proc_cls;
991 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
997 * Get a random item that has expired or has low priority.
998 * Call 'proc' with all values ZERO or NULL if the datastore is empty.
1000 * @param cls closure
1001 * @param proc function to call the value (once only).
1002 * @param proc_cls closure for proc
1005 sqlite_plugin_get_expiration (void *cls,
1006 PluginDatumProcessor proc, void *proc_cls)
1008 struct Plugin *plugin = cls;
1010 struct GNUNET_TIME_Absolute now;
1013 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1015 "Getting random block based on expiration and priority order.\n");
1017 now = GNUNET_TIME_absolute_get ();
1018 stmt = plugin->selExpi;
1019 if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value))
1021 LOG_SQLITE (plugin, NULL,
1022 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_bind_XXXX");
1023 if (SQLITE_OK != sqlite3_reset (stmt))
1024 LOG_SQLITE (plugin, NULL,
1025 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
1026 proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
1027 GNUNET_TIME_UNIT_ZERO_ABS, 0);
1030 execute_get (plugin, stmt, proc, proc_cls);
1037 * @param cls our plugin context
1040 sqlite_plugin_drop (void *cls)
1042 struct Plugin *plugin = cls;
1043 plugin->drop_on_shutdown = GNUNET_YES;
1048 * Get an estimate of how much space the database is
1051 * @param cls the 'struct Plugin'
1052 * @return the size of the database on disk (estimate)
1054 static unsigned long long
1055 sqlite_plugin_estimate_size (void *cls)
1057 struct Plugin *plugin = cls;
1065 if (SQLITE_VERSION_NUMBER < 3006000)
1067 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
1069 _("sqlite version to old to determine size, assuming zero\n"));
1073 sqlite3_exec (plugin->dbh,
1074 "VACUUM", NULL, NULL, ENULL));
1076 sqlite3_exec (plugin->dbh,
1077 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
1079 sq_prepare (plugin->dbh,
1080 "PRAGMA page_count",
1083 sqlite3_step (stmt))
1084 pages = sqlite3_column_int64 (stmt, 0);
1087 sqlite3_finalize (stmt);
1089 sq_prepare (plugin->dbh,
1092 CHECK (SQLITE_ROW ==
1093 sqlite3_step (stmt));
1094 page_size = sqlite3_column_int64 (stmt, 0);
1095 sqlite3_finalize (stmt);
1096 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1097 _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1098 (unsigned long long) pages,
1099 (unsigned long long) page_size);
1100 return pages * page_size;
1105 * Entry point for the plugin.
1107 * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*"
1108 * @return NULL on error, othrewise the plugin context
1111 libgnunet_plugin_datastore_sqlite_init (void *cls)
1113 static struct Plugin plugin;
1114 struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
1115 struct GNUNET_DATASTORE_PluginFunctions *api;
1117 if (plugin.env != NULL)
1118 return NULL; /* can only initialize once! */
1119 memset (&plugin, 0, sizeof(struct Plugin));
1122 database_setup (env->cfg, &plugin))
1124 database_shutdown (&plugin);
1127 api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions));
1129 api->estimate_size = &sqlite_plugin_estimate_size;
1130 api->put = &sqlite_plugin_put;
1131 api->update = &sqlite_plugin_update;
1132 api->get_key = &sqlite_plugin_get_key;
1133 api->get_replication = &sqlite_plugin_get_replication;
1134 api->get_expiration = &sqlite_plugin_get_expiration;
1135 api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
1136 api->drop = &sqlite_plugin_drop;
1137 GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
1138 "sqlite", _("Sqlite database running\n"));
1144 * Exit point from the plugin.
1146 * @param cls the plugin context (as returned by "init")
1147 * @return always NULL
1150 libgnunet_plugin_datastore_sqlite_done (void *cls)
1153 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1154 struct Plugin *plugin = api->cls;
1157 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1159 "sqlite plugin is done\n");
1163 if (plugin->drop_on_shutdown)
1164 fn = GNUNET_strdup (plugin->fn);
1166 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1168 "Shutting down database\n");
1170 database_shutdown (plugin);
1175 if (0 != UNLINK(fn))
1176 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1182 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1184 "sqlite plugin is finished\n");
1189 /* end of plugin_datastore_sqlite.c */