2 This file is part of GNUnet
3 Copyright (C) 2006, 2009, 2015 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file datacache/plugin_datacache_sqlite.c
23 * @brief sqlite for an implementation of a database backend for the datacache
24 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_datacache_plugin.h"
31 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-sqlite", __VA_ARGS__)
33 #define LOG_STRERROR_FILE(kind,op,fn) GNUNET_log_from_strerror_file (kind, "datacache-sqlite", op, fn)
37 * How much overhead do we assume per entry in the
40 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 32)
43 * Context for all functions in this plugin.
48 * Our execution environment.
50 struct GNUNET_DATACACHE_PluginEnvironment *env;
53 * Handle to the sqlite database.
58 * Filename used for the DB.
63 * Number of key-value pairs in the database.
65 unsigned int num_items;
70 * Log an error message at log-level @a level that indicates
71 * a failure of the command @a cmd with the error from the database @a db
73 * @param db database handle
74 * @param level log level
75 * @param cmd failed command
77 #define LOG_SQLITE(db, level, cmd) do { LOG (level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0)
81 * Execute SQL statement.
83 * @param db database handle
84 * @param cmd SQL command to execute
86 #define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { LOG (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0)
90 * @brief Prepare a SQL statement
92 * @param dbh database handle
93 * @param zsql SQL statement text
94 * @param[out] ppStmt set to the prepared statement
95 * @return 0 on success
98 sq_prepare (sqlite3 *dbh,
99 const char *zSql, /* SQL statement, UTF-8 encoded */
100 sqlite3_stmt **ppStmt)
101 { /* OUT: Statement handle */
104 return sqlite3_prepare (dbh, zSql, strlen (zSql), ppStmt,
105 (const char **) &dummy);
110 * Store an item in the datastore.
112 * @param cls closure (our `struct Plugin`)
113 * @param key key to store data under
114 * @param size number of bytes in @a data
115 * @param data data to store
116 * @param type type of the value
117 * @param discard_time when to discard the value in any case
118 * @param path_info_len number of entries in @a path_info
119 * @param path_info array of peers that have processed the request
120 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
123 sqlite_plugin_put (void *cls,
124 const struct GNUNET_HashCode *key,
127 enum GNUNET_BLOCK_Type type,
128 struct GNUNET_TIME_Absolute discard_time,
129 unsigned int path_info_len,
130 const struct GNUNET_PeerIdentity *path_info)
132 struct Plugin *plugin = cls;
136 LOG (GNUNET_ERROR_TYPE_DEBUG,
137 "Processing PUT of %u bytes with key `%4s' and expiration %s\n",
140 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time), GNUNET_YES));
141 dval = (int64_t) discard_time.abs_value_us;
146 "INSERT INTO ds090 (type, expire, key, value, path) VALUES (?, ?, ?, ?, ?)",
149 LOG_SQLITE (plugin->dbh,
150 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
154 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
155 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, dval)) ||
157 sqlite3_bind_blob (stmt, 3,
158 key, sizeof (struct GNUNET_HashCode),
159 SQLITE_TRANSIENT)) ||
160 (SQLITE_OK != sqlite3_bind_blob (stmt, 4,
162 SQLITE_TRANSIENT)) ||
163 (SQLITE_OK != sqlite3_bind_blob (stmt, 5,
165 path_info_len * sizeof (struct GNUNET_PeerIdentity),
168 LOG_SQLITE (plugin->dbh,
169 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
171 sqlite3_finalize (stmt);
174 if (SQLITE_DONE != sqlite3_step (stmt))
176 LOG_SQLITE (plugin->dbh,
177 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
179 sqlite3_finalize (stmt);
183 if (SQLITE_OK != sqlite3_finalize (stmt))
184 LOG_SQLITE (plugin->dbh,
185 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
187 return size + OVERHEAD;
192 * Iterate over the results for a particular key
195 * @param cls closure (our `struct Plugin`)
197 * @param type entries of which type are relevant?
198 * @param iter maybe NULL (to just count)
199 * @param iter_cls closure for @a iter
200 * @return the number of results found
203 sqlite_plugin_get (void *cls,
204 const struct GNUNET_HashCode *key,
205 enum GNUNET_BLOCK_Type type,
206 GNUNET_DATACACHE_Iterator iter,
209 struct Plugin *plugin = cls;
211 struct GNUNET_TIME_Absolute now;
212 struct GNUNET_TIME_Absolute exp;
221 const struct GNUNET_PeerIdentity *path;
223 now = GNUNET_TIME_absolute_get ();
224 LOG (GNUNET_ERROR_TYPE_DEBUG,
225 "Processing GET for key `%4s'\n",
229 "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?",
232 LOG_SQLITE (plugin->dbh,
233 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
237 ntime = (int64_t) now.abs_value_us;
238 GNUNET_assert (ntime >= 0);
240 sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
241 SQLITE_TRANSIENT)) ||
242 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
243 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
245 LOG_SQLITE (plugin->dbh,
246 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
248 sqlite3_finalize (stmt);
252 if (SQLITE_ROW != sqlite3_step (stmt))
254 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
256 sqlite3_finalize (stmt);
257 LOG (GNUNET_ERROR_TYPE_DEBUG,
258 "No content found when processing GET for key `%4s'\n",
262 total = sqlite3_column_int (stmt, 0);
263 sqlite3_finalize (stmt);
264 if ((total == 0) || (iter == NULL))
267 LOG (GNUNET_ERROR_TYPE_DEBUG,
268 "No content found when processing GET for key `%4s'\n",
274 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
277 off = (off + 1) % total;
278 GNUNET_snprintf (scratch, sizeof (scratch),
279 "SELECT value,expire,path FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
281 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
283 LOG_SQLITE (plugin->dbh,
284 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
289 sqlite3_bind_blob (stmt, 1,
291 sizeof (struct GNUNET_HashCode),
292 SQLITE_TRANSIENT)) ||
293 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
294 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
296 LOG_SQLITE (plugin->dbh,
297 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
299 sqlite3_finalize (stmt);
302 if (sqlite3_step (stmt) != SQLITE_ROW)
304 size = sqlite3_column_bytes (stmt, 0);
305 dat = sqlite3_column_blob (stmt, 0);
306 exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
307 psize = sqlite3_column_bytes (stmt, 2);
308 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
313 psize /= sizeof (struct GNUNET_PeerIdentity);
315 path = sqlite3_column_blob (stmt, 2);
318 ntime = (int64_t) exp.abs_value_us;
319 if (ntime == INT64_MAX)
320 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
322 LOG (GNUNET_ERROR_TYPE_DEBUG,
323 "Found %u-byte result when processing GET for key `%4s'\n",
326 if (GNUNET_OK != iter (iter_cls, key, size, dat, type, exp, psize, path))
328 sqlite3_finalize (stmt);
331 sqlite3_finalize (stmt);
338 * Delete the entry with the lowest expiration value
339 * from the datacache right now.
341 * @param cls closure (our `struct Plugin`)
342 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
345 sqlite_plugin_del (void *cls)
347 struct Plugin *plugin = cls;
348 unsigned long long rowid;
352 struct GNUNET_HashCode hc;
354 LOG (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s'\n", "DEL");
359 "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
362 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
365 (void) sqlite3_finalize (stmt);
366 return GNUNET_SYSERR;
368 if (SQLITE_ROW != sqlite3_step (stmt))
370 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
372 (void) sqlite3_finalize (stmt);
373 return GNUNET_SYSERR;
375 rowid = sqlite3_column_int64 (stmt, 0);
376 GNUNET_assert (sqlite3_column_bytes (stmt, 1) == sizeof (struct GNUNET_HashCode));
377 memcpy (&hc, sqlite3_column_blob (stmt, 1), sizeof (struct GNUNET_HashCode));
378 dsize = sqlite3_column_bytes (stmt, 2);
379 if (SQLITE_OK != sqlite3_finalize (stmt))
380 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
382 if (sq_prepare (plugin->dbh, "DELETE FROM ds090 WHERE _ROWID_=?", &dstmt) !=
385 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
388 (void) sqlite3_finalize (stmt);
389 return GNUNET_SYSERR;
391 if (SQLITE_OK != sqlite3_bind_int64 (dstmt, 1, rowid))
393 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
395 (void) sqlite3_finalize (dstmt);
396 return GNUNET_SYSERR;
398 if (sqlite3_step (dstmt) != SQLITE_DONE)
400 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
402 (void) sqlite3_finalize (dstmt);
403 return GNUNET_SYSERR;
406 plugin->env->delete_notify (plugin->env->cls, &hc, dsize + OVERHEAD);
407 if (SQLITE_OK != sqlite3_finalize (dstmt))
408 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
415 * Obtain a random key-value pair from the datacache.
417 * @param cls closure (our `struct Plugin`)
418 * @param iter maybe NULL (to just count)
419 * @param iter_cls closure for @a iter
420 * @return the number of results found, zero (datacache empty) or one
423 sqlite_plugin_get_random (void *cls,
424 GNUNET_DATACACHE_Iterator iter,
427 struct Plugin *plugin = cls;
429 struct GNUNET_TIME_Absolute exp;
437 const struct GNUNET_PeerIdentity *path;
438 const struct GNUNET_HashCode *key;
440 if (0 == plugin->num_items)
442 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
444 GNUNET_snprintf (scratch,
446 "SELECT value,expire,path,key,type FROM ds090 ORDER BY key LIMIT 1 OFFSET %u",
449 sq_prepare (plugin->dbh, scratch, &stmt))
451 LOG_SQLITE (plugin->dbh,
452 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
456 if (SQLITE_ROW != sqlite3_step (stmt))
459 sqlite3_finalize (stmt);
462 size = sqlite3_column_bytes (stmt, 0);
463 dat = sqlite3_column_blob (stmt, 0);
464 exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
465 psize = sqlite3_column_bytes (stmt, 2);
466 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
471 psize /= sizeof (struct GNUNET_PeerIdentity);
473 path = sqlite3_column_blob (stmt, 2);
477 GNUNET_assert (sizeof (struct GNUNET_HashCode) ==
478 sqlite3_column_bytes (stmt, 3));
479 key = sqlite3_column_blob (stmt, 3);
480 type = sqlite3_column_int (stmt, 4);
482 ntime = (int64_t) exp.abs_value_us;
483 if (ntime == INT64_MAX)
484 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
485 LOG (GNUNET_ERROR_TYPE_DEBUG,
486 "Found %u-byte result with key %s when processing GET-RANDOM\n",
489 (void) iter (iter_cls,
497 sqlite3_finalize (stmt);
503 * Entry point for the plugin.
505 * @param cls closure (the `struct GNUNET_DATACACHE_PluginEnvironment`)
506 * @return the plugin's closure (our `struct Plugin`)
509 libgnunet_plugin_datacache_sqlite_init (void *cls)
511 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
512 struct GNUNET_DATACACHE_PluginFunctions *api;
513 struct Plugin *plugin;
520 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
524 if (SQLITE_OK != sqlite3_open (":memory:", &dbh))
530 fn = GNUNET_DISK_mktemp ("gnunet-datacache");
536 /* fn should be UTF-8-encoded. If it isn't, it's a bug. */
537 fn_utf8 = GNUNET_strdup (fn);
538 if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))
541 GNUNET_free (fn_utf8);
547 SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY");
548 SQLITE3_EXEC (dbh, "PRAGMA locking_mode=EXCLUSIVE");
549 SQLITE3_EXEC (dbh, "PRAGMA journal_mode=OFF");
550 SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF");
551 SQLITE3_EXEC (dbh, "PRAGMA page_size=4092");
553 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
556 SQLITE3_EXEC (dbh, "PRAGMA sqlite_temp_store=3");
559 "CREATE TABLE ds090 (" " type INTEGER NOT NULL DEFAULT 0,"
560 " expire INTEGER NOT NULL DEFAULT 0,"
561 " key BLOB NOT NULL DEFAULT '',"
562 " value BLOB NOT NULL DEFAULT '',"
563 " path BLOB DEFAULT '')");
564 SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds090 (key,type,expire)");
565 SQLITE3_EXEC (dbh, "CREATE INDEX idx_expire ON ds090 (expire)");
566 plugin = GNUNET_new (struct Plugin);
569 plugin->fn = fn_utf8;
570 api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
572 api->get = &sqlite_plugin_get;
573 api->put = &sqlite_plugin_put;
574 api->del = &sqlite_plugin_del;
575 api->get_random = &sqlite_plugin_get_random;
576 LOG (GNUNET_ERROR_TYPE_INFO,
577 "Sqlite datacache running\n");
583 * Exit point from the plugin.
585 * @param cls closure (our `struct Plugin`)
589 libgnunet_plugin_datacache_sqlite_done (void *cls)
591 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
592 struct Plugin *plugin = api->cls;
595 #if SQLITE_VERSION_NUMBER >= 3007000
599 #if !WINDOWS || defined(__CYGWIN__)
600 if ( (NULL != plugin->fn) &&
601 (0 != UNLINK (plugin->fn)) )
602 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", plugin->fn);
603 GNUNET_free_non_null (plugin->fn);
605 result = sqlite3_close (plugin->dbh);
606 #if SQLITE_VERSION_NUMBER >= 3007000
607 if (result == SQLITE_BUSY)
609 LOG (GNUNET_ERROR_TYPE_WARNING,
611 ("Tried to close sqlite without finalizing all prepared statements.\n"));
612 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
615 LOG (GNUNET_ERROR_TYPE_DEBUG, "Closing statement %p\n", stmt);
616 result = sqlite3_finalize (stmt);
617 if (result != SQLITE_OK)
618 LOG (GNUNET_ERROR_TYPE_WARNING, _("Failed to close statement %p: %d\n"),
620 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
622 result = sqlite3_close (plugin->dbh);
625 if (SQLITE_OK != result)
626 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
628 #if WINDOWS && !defined(__CYGWIN__)
629 if ( (NULL != plugin->fn) &&
630 (0 != UNLINK (plugin->fn)) )
631 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", plugin->fn);
632 GNUNET_free_non_null (plugin->fn);
634 GNUNET_free (plugin);
641 /* end of plugin_datacache_sqlite.c */