2 This file is part of GNUnet
3 Copyright (C) 2006, 2009, 2015 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
22 * @file datacache/plugin_datacache_sqlite.c
23 * @brief sqlite for an implementation of a database backend for the datacache
24 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_datacache_plugin.h"
31 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-sqlite", __VA_ARGS__)
33 #define LOG_STRERROR_FILE(kind,op,fn) GNUNET_log_from_strerror_file (kind, "datacache-sqlite", op, fn)
37 * How much overhead do we assume per entry in the
40 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 32)
43 * Context for all functions in this plugin.
48 * Our execution environment.
50 struct GNUNET_DATACACHE_PluginEnvironment *env;
53 * Handle to the sqlite database.
58 * Filename used for the DB.
63 * Number of key-value pairs in the database.
65 unsigned int num_items;
70 * Log an error message at log-level @a level that indicates
71 * a failure of the command @a cmd with the error from the database @a db
73 * @param db database handle
74 * @param level log level
75 * @param cmd failed command
77 #define LOG_SQLITE(db, level, cmd) do { LOG (level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0)
81 * Execute SQL statement.
83 * @param db database handle
84 * @param cmd SQL command to execute
86 #define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { LOG (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0)
90 * @brief Prepare a SQL statement
92 * @param dbh database handle
93 * @param zsql SQL statement text
94 * @param[out] ppStmt set to the prepared statement
95 * @return 0 on success
98 sq_prepare (sqlite3 *dbh,
99 const char *zSql, /* SQL statement, UTF-8 encoded */
100 sqlite3_stmt **ppStmt)
101 { /* OUT: Statement handle */
104 return sqlite3_prepare (dbh,
107 (const char **) &dummy);
112 * Store an item in the datastore.
114 * @param cls closure (our `struct Plugin`)
115 * @param key key to store @a data under
116 * @param size number of bytes in @a data
117 * @param data data to store
118 * @param type type of the value
119 * @param discard_time when to discard the value in any case
120 * @param path_info_len number of entries in @a path_info
121 * @param path_info array of peers that have processed the request
122 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
125 sqlite_plugin_put (void *cls,
126 const struct GNUNET_HashCode *key,
129 enum GNUNET_BLOCK_Type type,
130 struct GNUNET_TIME_Absolute discard_time,
131 unsigned int path_info_len,
132 const struct GNUNET_PeerIdentity *path_info)
134 struct Plugin *plugin = cls;
138 LOG (GNUNET_ERROR_TYPE_DEBUG,
139 "Processing PUT of %u bytes with key `%4s' and expiration %s\n",
142 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time), GNUNET_YES));
143 dval = (int64_t) discard_time.abs_value_us;
148 "INSERT INTO ds090 (type, expire, key, value, path) VALUES (?, ?, ?, ?, ?)",
151 LOG_SQLITE (plugin->dbh,
152 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
156 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
157 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, dval)) ||
159 sqlite3_bind_blob (stmt, 3,
160 key, sizeof (struct GNUNET_HashCode),
161 SQLITE_TRANSIENT)) ||
162 (SQLITE_OK != sqlite3_bind_blob (stmt, 4,
164 SQLITE_TRANSIENT)) ||
165 (SQLITE_OK != sqlite3_bind_blob (stmt, 5,
167 path_info_len * sizeof (struct GNUNET_PeerIdentity),
170 LOG_SQLITE (plugin->dbh,
171 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
173 sqlite3_finalize (stmt);
176 if (SQLITE_DONE != sqlite3_step (stmt))
178 LOG_SQLITE (plugin->dbh,
179 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
181 sqlite3_finalize (stmt);
185 if (SQLITE_OK != sqlite3_finalize (stmt))
186 LOG_SQLITE (plugin->dbh,
187 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
189 return size + OVERHEAD;
194 * Iterate over the results for a particular key
197 * @param cls closure (our `struct Plugin`)
199 * @param type entries of which type are relevant?
200 * @param iter maybe NULL (to just count)
201 * @param iter_cls closure for @a iter
202 * @return the number of results found
205 sqlite_plugin_get (void *cls,
206 const struct GNUNET_HashCode *key,
207 enum GNUNET_BLOCK_Type type,
208 GNUNET_DATACACHE_Iterator iter,
211 struct Plugin *plugin = cls;
213 struct GNUNET_TIME_Absolute now;
214 struct GNUNET_TIME_Absolute exp;
223 const struct GNUNET_PeerIdentity *path;
225 now = GNUNET_TIME_absolute_get ();
226 LOG (GNUNET_ERROR_TYPE_DEBUG,
227 "Processing GET for key `%4s'\n",
231 "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?",
234 LOG_SQLITE (plugin->dbh,
235 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
239 ntime = (int64_t) now.abs_value_us;
240 GNUNET_assert (ntime >= 0);
242 sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
243 SQLITE_TRANSIENT)) ||
244 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
245 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
247 LOG_SQLITE (plugin->dbh,
248 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
250 sqlite3_finalize (stmt);
254 if (SQLITE_ROW != sqlite3_step (stmt))
256 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
258 sqlite3_finalize (stmt);
259 LOG (GNUNET_ERROR_TYPE_DEBUG,
260 "No content found when processing GET for key `%4s'\n",
264 total = sqlite3_column_int (stmt, 0);
265 sqlite3_finalize (stmt);
266 if ((0 == total) || (NULL == iter))
269 LOG (GNUNET_ERROR_TYPE_DEBUG,
270 "No content found when processing GET for key `%4s'\n",
276 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
279 off = (off + 1) % total;
280 GNUNET_snprintf (scratch, sizeof (scratch),
281 "SELECT value,expire,path FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
283 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
285 LOG_SQLITE (plugin->dbh,
286 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
291 sqlite3_bind_blob (stmt, 1,
293 sizeof (struct GNUNET_HashCode),
294 SQLITE_TRANSIENT)) ||
295 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
296 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
298 LOG_SQLITE (plugin->dbh,
299 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
301 sqlite3_finalize (stmt);
304 if (sqlite3_step (stmt) != SQLITE_ROW)
306 size = sqlite3_column_bytes (stmt, 0);
307 dat = sqlite3_column_blob (stmt, 0);
308 exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
309 psize = sqlite3_column_bytes (stmt, 2);
310 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
315 psize /= sizeof (struct GNUNET_PeerIdentity);
317 path = sqlite3_column_blob (stmt, 2);
320 ntime = (int64_t) exp.abs_value_us;
321 if (ntime == INT64_MAX)
322 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
324 LOG (GNUNET_ERROR_TYPE_DEBUG,
325 "Found %u-byte result when processing GET for key `%4s'\n",
328 if (GNUNET_OK != iter (iter_cls,
337 sqlite3_finalize (stmt);
340 sqlite3_finalize (stmt);
347 * Delete the entry with the lowest expiration value
348 * from the datacache right now.
350 * @param cls closure (our `struct Plugin`)
351 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
354 sqlite_plugin_del (void *cls)
356 struct Plugin *plugin = cls;
357 unsigned long long rowid;
361 struct GNUNET_HashCode hc;
363 LOG (GNUNET_ERROR_TYPE_DEBUG,
368 sq_prepare (plugin->dbh,
369 "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
372 LOG_SQLITE (plugin->dbh,
373 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
376 (void) sqlite3_finalize (stmt);
377 return GNUNET_SYSERR;
379 if (SQLITE_ROW != sqlite3_step (stmt))
381 LOG_SQLITE (plugin->dbh,
382 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
384 (void) sqlite3_finalize (stmt);
385 return GNUNET_SYSERR;
387 rowid = sqlite3_column_int64 (stmt, 0);
388 GNUNET_assert (sqlite3_column_bytes (stmt, 1) == sizeof (struct GNUNET_HashCode));
389 memcpy (&hc, sqlite3_column_blob (stmt, 1), sizeof (struct GNUNET_HashCode));
390 dsize = sqlite3_column_bytes (stmt, 2);
391 if (SQLITE_OK != sqlite3_finalize (stmt))
392 LOG_SQLITE (plugin->dbh,
393 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
396 sq_prepare (plugin->dbh,
397 "DELETE FROM ds090 WHERE _ROWID_=?", &dstmt))
399 LOG_SQLITE (plugin->dbh,
400 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
403 (void) sqlite3_finalize (stmt);
404 return GNUNET_SYSERR;
406 if (SQLITE_OK != sqlite3_bind_int64 (dstmt, 1, rowid))
408 LOG_SQLITE (plugin->dbh,
409 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
411 (void) sqlite3_finalize (dstmt);
412 return GNUNET_SYSERR;
414 if (SQLITE_DONE != sqlite3_step (dstmt))
416 LOG_SQLITE (plugin->dbh,
417 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
419 (void) sqlite3_finalize (dstmt);
420 return GNUNET_SYSERR;
423 plugin->env->delete_notify (plugin->env->cls,
426 if (SQLITE_OK != sqlite3_finalize (dstmt))
427 LOG_SQLITE (plugin->dbh,
428 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
435 * Obtain a random key-value pair from the datacache.
437 * @param cls closure (our `struct Plugin`)
438 * @param iter maybe NULL (to just count)
439 * @param iter_cls closure for @a iter
440 * @return the number of results found, zero (datacache empty) or one
443 sqlite_plugin_get_random (void *cls,
444 GNUNET_DATACACHE_Iterator iter,
447 struct Plugin *plugin = cls;
449 struct GNUNET_TIME_Absolute exp;
457 const struct GNUNET_PeerIdentity *path;
458 const struct GNUNET_HashCode *key;
460 if (0 == plugin->num_items)
464 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
466 GNUNET_snprintf (scratch,
468 "SELECT value,expire,path,key,type FROM ds090 ORDER BY key LIMIT 1 OFFSET %u",
471 sq_prepare (plugin->dbh, scratch, &stmt))
473 LOG_SQLITE (plugin->dbh,
474 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
478 if (SQLITE_ROW != sqlite3_step (stmt))
481 sqlite3_finalize (stmt);
484 size = sqlite3_column_bytes (stmt, 0);
485 dat = sqlite3_column_blob (stmt, 0);
486 exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
487 psize = sqlite3_column_bytes (stmt, 2);
488 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
493 psize /= sizeof (struct GNUNET_PeerIdentity);
495 path = sqlite3_column_blob (stmt, 2);
499 GNUNET_assert (sizeof (struct GNUNET_HashCode) ==
500 sqlite3_column_bytes (stmt, 3));
501 key = sqlite3_column_blob (stmt, 3);
502 type = sqlite3_column_int (stmt, 4);
504 ntime = (int64_t) exp.abs_value_us;
505 if (ntime == INT64_MAX)
506 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
507 LOG (GNUNET_ERROR_TYPE_DEBUG,
508 "Found %u-byte result with key %s when processing GET-RANDOM\n",
511 (void) iter (iter_cls,
519 sqlite3_finalize (stmt);
525 * Iterate over the results that are "close" to a particular key in
526 * the datacache. "close" is defined as numerically larger than @a
527 * key (when interpreted as a circular address space), with small
530 * @param cls closure (internal context for the plugin)
531 * @param key area of the keyspace to look into
532 * @param num_results number of results that should be returned to @a iter
533 * @param iter maybe NULL (to just count)
534 * @param iter_cls closure for @a iter
535 * @return the number of results found
538 sqlite_plugin_get_closest (void *cls,
539 const struct GNUNET_HashCode *key,
540 unsigned int num_results,
541 GNUNET_DATACACHE_Iterator iter,
544 struct Plugin *plugin = cls;
546 struct GNUNET_TIME_Absolute now;
547 struct GNUNET_TIME_Absolute exp;
554 const struct GNUNET_PeerIdentity *path;
556 now = GNUNET_TIME_absolute_get ();
557 LOG (GNUNET_ERROR_TYPE_DEBUG,
558 "Processing GET_CLOSEST for key `%4s'\n",
561 sq_prepare (plugin->dbh,
562 "SELECT value,expire,path,type,key FROM ds090 WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?",
565 LOG_SQLITE (plugin->dbh,
566 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
570 ntime = (int64_t) now.abs_value_us;
571 GNUNET_assert (ntime >= 0);
573 sqlite3_bind_blob (stmt,
576 sizeof (struct GNUNET_HashCode),
577 SQLITE_TRANSIENT)) ||
578 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, now.abs_value_us)) ||
579 (SQLITE_OK != sqlite3_bind_int (stmt, 3, num_results)) )
581 LOG_SQLITE (plugin->dbh,
582 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
584 sqlite3_finalize (stmt);
588 while (SQLITE_ROW == sqlite3_step (stmt))
590 if (sizeof (struct GNUNET_HashCode) !=
591 sqlite3_column_bytes (stmt, 4))
596 size = sqlite3_column_bytes (stmt, 0);
597 dat = sqlite3_column_blob (stmt, 0);
598 exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
599 psize = sqlite3_column_bytes (stmt, 2);
600 type = sqlite3_column_int (stmt, 3);
601 key = sqlite3_column_blob (stmt, 4);
602 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
607 psize /= sizeof (struct GNUNET_PeerIdentity);
609 path = sqlite3_column_blob (stmt, 2);
612 ntime = (int64_t) exp.abs_value_us;
613 if (ntime == INT64_MAX)
614 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
616 LOG (GNUNET_ERROR_TYPE_DEBUG,
617 "Found %u-byte result at %s when processing GET_CLOSE\n",
620 if (GNUNET_OK != iter (iter_cls,
629 sqlite3_finalize (stmt);
633 sqlite3_finalize (stmt);
639 * Entry point for the plugin.
641 * @param cls closure (the `struct GNUNET_DATACACHE_PluginEnvironment`)
642 * @return the plugin's closure (our `struct Plugin`)
645 libgnunet_plugin_datacache_sqlite_init (void *cls)
647 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
648 struct GNUNET_DATACACHE_PluginFunctions *api;
649 struct Plugin *plugin;
656 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
660 if (SQLITE_OK != sqlite3_open (":memory:", &dbh))
666 fn = GNUNET_DISK_mktemp ("gnunet-datacache");
672 /* fn should be UTF-8-encoded. If it isn't, it's a bug. */
673 fn_utf8 = GNUNET_strdup (fn);
674 if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))
677 GNUNET_free (fn_utf8);
683 SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY");
684 SQLITE3_EXEC (dbh, "PRAGMA locking_mode=EXCLUSIVE");
685 SQLITE3_EXEC (dbh, "PRAGMA journal_mode=OFF");
686 SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF");
687 SQLITE3_EXEC (dbh, "PRAGMA page_size=4092");
689 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
692 SQLITE3_EXEC (dbh, "PRAGMA sqlite_temp_store=3");
695 "CREATE TABLE ds090 (" " type INTEGER NOT NULL DEFAULT 0,"
696 " expire INTEGER NOT NULL DEFAULT 0,"
697 " key BLOB NOT NULL DEFAULT '',"
698 " value BLOB NOT NULL DEFAULT '',"
699 " path BLOB DEFAULT '')");
700 SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds090 (key,type,expire)");
701 SQLITE3_EXEC (dbh, "CREATE INDEX idx_expire ON ds090 (expire)");
702 plugin = GNUNET_new (struct Plugin);
705 plugin->fn = fn_utf8;
706 api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
708 api->get = &sqlite_plugin_get;
709 api->put = &sqlite_plugin_put;
710 api->del = &sqlite_plugin_del;
711 api->get_random = &sqlite_plugin_get_random;
712 api->get_closest = &sqlite_plugin_get_closest;
713 LOG (GNUNET_ERROR_TYPE_INFO,
714 "Sqlite datacache running\n");
720 * Exit point from the plugin.
722 * @param cls closure (our `struct Plugin`)
726 libgnunet_plugin_datacache_sqlite_done (void *cls)
728 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
729 struct Plugin *plugin = api->cls;
732 #if SQLITE_VERSION_NUMBER >= 3007000
736 #if !WINDOWS || defined(__CYGWIN__)
737 if ( (NULL != plugin->fn) &&
738 (0 != UNLINK (plugin->fn)) )
739 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
742 GNUNET_free_non_null (plugin->fn);
744 result = sqlite3_close (plugin->dbh);
745 #if SQLITE_VERSION_NUMBER >= 3007000
746 if (SQLITE_BUSY == result)
748 LOG (GNUNET_ERROR_TYPE_WARNING,
749 _("Tried to close sqlite without finalizing all prepared statements.\n"));
750 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
753 result = sqlite3_finalize (stmt);
754 if (result != SQLITE_OK)
755 LOG (GNUNET_ERROR_TYPE_WARNING,
756 "Failed to close statement %p: %d\n",
759 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
761 result = sqlite3_close (plugin->dbh);
764 if (SQLITE_OK != result)
765 LOG_SQLITE (plugin->dbh,
766 GNUNET_ERROR_TYPE_ERROR,
769 #if WINDOWS && !defined(__CYGWIN__)
770 if ( (NULL != plugin->fn) &&
771 (0 != UNLINK (plugin->fn)) )
772 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
775 GNUNET_free_non_null (plugin->fn);
777 GNUNET_free (plugin);
784 /* end of plugin_datacache_sqlite.c */