2 This file is part of GNUnet
3 (C) 2006, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file datacache/plugin_datacache_sqlite.c
23 * @brief sqlite for an implementation of a database backend for the datacache
24 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_datacache_plugin.h"
31 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-sqlite", __VA_ARGS__)
33 #define LOG_STRERROR_FILE(kind,op,fn) GNUNET_log_from_strerror_file (kind, "datacache-sqlite", op, fn)
37 * How much overhead do we assume per entry in the
40 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 32)
43 * Context for all functions in this plugin.
48 * Our execution environment.
50 struct GNUNET_DATACACHE_PluginEnvironment *env;
53 * Handle to the sqlite database.
58 * Filename used for the DB.
65 * Log an error message at log-level 'level' that indicates
66 * a failure of the command 'cmd' on file 'filename'
67 * with the message given by strerror(errno).
69 #define LOG_SQLITE(db, level, cmd) do { LOG (level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0)
72 #define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { LOG (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0)
76 * @brief Prepare a SQL statement
79 sq_prepare (sqlite3 * dbh, const char *zSql, /* SQL statement, UTF-8 encoded */
80 sqlite3_stmt ** ppStmt)
81 { /* OUT: Statement handle */
84 return sqlite3_prepare (dbh, zSql, strlen (zSql), ppStmt,
85 (const char **) &dummy);
90 * Store an item in the datastore.
92 * @param cls closure (our "struct Plugin")
93 * @param key key to store data under
94 * @param size number of bytes in data
95 * @param data data to store
96 * @param type type of the value
97 * @param discard_time when to discard the value in any case
98 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
101 sqlite_plugin_put (void *cls,
102 const struct GNUNET_HashCode *key,
103 size_t size, const char *data,
104 enum GNUNET_BLOCK_Type type,
105 struct GNUNET_TIME_Absolute discard_time,
106 unsigned int path_info_len,
107 const struct GNUNET_PeerIdentity *path_info)
109 struct Plugin *plugin = cls;
113 LOG (GNUNET_ERROR_TYPE_DEBUG,
114 "Processing `%s' of %u bytes with key `%4s' and expiration %s\n",
115 "PUT", (unsigned int) size, GNUNET_h2s (key),
116 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time), GNUNET_YES));
117 dval = (int64_t) discard_time.abs_value;
122 "INSERT INTO ds090 (type, expire, key, value, path) VALUES (?, ?, ?, ?, ?)",
125 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
129 if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
130 (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, dval)) ||
132 sqlite3_bind_blob (stmt, 3,
133 key, sizeof (struct GNUNET_HashCode),
134 SQLITE_TRANSIENT)) ||
135 (SQLITE_OK != sqlite3_bind_blob (stmt, 4,
137 SQLITE_TRANSIENT)) ||
138 (SQLITE_OK != sqlite3_bind_blob (stmt, 5,
140 path_info_len * sizeof (struct GNUNET_PeerIdentity),
143 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
145 sqlite3_finalize (stmt);
148 if (SQLITE_DONE != sqlite3_step (stmt))
150 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
152 sqlite3_finalize (stmt);
155 if (SQLITE_OK != sqlite3_finalize (stmt))
156 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
158 return size + OVERHEAD;
163 * Iterate over the results for a particular key
166 * @param cls closure (our "struct Plugin")
168 * @param type entries of which type are relevant?
169 * @param iter maybe NULL (to just count)
170 * @param iter_cls closure for iter
171 * @return the number of results found
174 sqlite_plugin_get (void *cls, const struct GNUNET_HashCode * key,
175 enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter,
178 struct Plugin *plugin = cls;
180 struct GNUNET_TIME_Absolute now;
181 struct GNUNET_TIME_Absolute exp;
190 const struct GNUNET_PeerIdentity *path;
192 now = GNUNET_TIME_absolute_get ();
193 LOG (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' for key `%4s'\n", "GET",
197 "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?",
200 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
204 ntime = (int64_t) now.abs_value;
205 GNUNET_assert (ntime >= 0);
207 sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
208 SQLITE_TRANSIENT)) ||
209 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
210 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value)))
212 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
214 sqlite3_finalize (stmt);
218 if (SQLITE_ROW != sqlite3_step (stmt))
220 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
222 sqlite3_finalize (stmt);
223 LOG (GNUNET_ERROR_TYPE_DEBUG,
224 "No content found when processing `%s' for key `%4s'\n", "GET",
228 total = sqlite3_column_int (stmt, 0);
229 sqlite3_finalize (stmt);
230 if ((total == 0) || (iter == NULL))
233 LOG (GNUNET_ERROR_TYPE_DEBUG,
234 "No content found when processing `%s' for key `%4s'\n", "GET",
240 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
243 off = (off + 1) % total;
244 GNUNET_snprintf (scratch, sizeof (scratch),
245 "SELECT value,expire,path FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
247 if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
249 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
254 sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
255 SQLITE_TRANSIENT)) ||
256 (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
257 (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value)))
259 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
261 sqlite3_finalize (stmt);
264 if (sqlite3_step (stmt) != SQLITE_ROW)
266 size = sqlite3_column_bytes (stmt, 0);
267 dat = sqlite3_column_blob (stmt, 0);
268 exp.abs_value = sqlite3_column_int64 (stmt, 1);
269 psize = sqlite3_column_bytes (stmt, 2);
270 if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
275 psize /= sizeof (struct GNUNET_PeerIdentity);
277 path = sqlite3_column_blob (stmt, 2);
280 ntime = (int64_t) exp.abs_value;
281 if (ntime == INT64_MAX)
282 exp = GNUNET_TIME_UNIT_FOREVER_ABS;
284 LOG (GNUNET_ERROR_TYPE_DEBUG,
285 "Found %u-byte result when processing `%s' for key `%4s'\n",
286 (unsigned int) size, "GET", GNUNET_h2s (key));
287 if (GNUNET_OK != iter (iter_cls, key, size, dat, type, exp, psize, path))
289 sqlite3_finalize (stmt);
292 sqlite3_finalize (stmt);
299 * Delete the entry with the lowest expiration value
300 * from the datacache right now.
302 * @param cls closure (our "struct Plugin")
303 * @return GNUNET_OK on success, GNUNET_SYSERR on error
306 sqlite_plugin_del (void *cls)
308 struct Plugin *plugin = cls;
309 unsigned long long rowid;
313 struct GNUNET_HashCode hc;
315 LOG (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s'\n", "DEL");
320 "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
323 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
326 (void) sqlite3_finalize (stmt);
327 return GNUNET_SYSERR;
329 if (SQLITE_ROW != sqlite3_step (stmt))
331 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
333 (void) sqlite3_finalize (stmt);
334 return GNUNET_SYSERR;
336 rowid = sqlite3_column_int64 (stmt, 0);
337 GNUNET_assert (sqlite3_column_bytes (stmt, 1) == sizeof (struct GNUNET_HashCode));
338 memcpy (&hc, sqlite3_column_blob (stmt, 1), sizeof (struct GNUNET_HashCode));
339 dsize = sqlite3_column_bytes (stmt, 2);
340 if (SQLITE_OK != sqlite3_finalize (stmt))
341 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
343 if (sq_prepare (plugin->dbh, "DELETE FROM ds090 WHERE _ROWID_=?", &dstmt) !=
346 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
349 (void) sqlite3_finalize (stmt);
350 return GNUNET_SYSERR;
352 if (SQLITE_OK != sqlite3_bind_int64 (dstmt, 1, rowid))
354 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
356 (void) sqlite3_finalize (dstmt);
357 return GNUNET_SYSERR;
359 if (sqlite3_step (dstmt) != SQLITE_DONE)
361 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
363 (void) sqlite3_finalize (dstmt);
364 return GNUNET_SYSERR;
366 plugin->env->delete_notify (plugin->env->cls, &hc, dsize + OVERHEAD);
367 if (SQLITE_OK != sqlite3_finalize (dstmt))
368 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
375 * Entry point for the plugin.
377 * @param cls closure (the "struct GNUNET_DATACACHE_PluginEnvironmnet")
378 * @return the plugin's closure (our "struct Plugin")
381 libgnunet_plugin_datacache_sqlite_init (void *cls)
383 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
384 struct GNUNET_DATACACHE_PluginFunctions *api;
385 struct Plugin *plugin;
392 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
396 if (SQLITE_OK != sqlite3_open (":memory:", &dbh))
402 fn = GNUNET_DISK_mktemp ("gnunet-datacache");
408 /* fn should be UTF-8-encoded. If it isn't, it's a bug. */
409 fn_utf8 = GNUNET_strdup (fn);
410 if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))
413 GNUNET_free (fn_utf8);
419 SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY");
420 SQLITE3_EXEC (dbh, "PRAGMA locking_mode=EXCLUSIVE");
421 SQLITE3_EXEC (dbh, "PRAGMA journal_mode=OFF");
422 SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF");
423 SQLITE3_EXEC (dbh, "PRAGMA count_changes=OFF");
424 SQLITE3_EXEC (dbh, "PRAGMA page_size=4092");
426 GNUNET_CONFIGURATION_get_value_yesno (env->cfg,
429 SQLITE3_EXEC (dbh, "PRAGMA sqlite_temp_store=3");
432 "CREATE TABLE ds090 (" " type INTEGER NOT NULL DEFAULT 0,"
433 " expire INTEGER NOT NULL DEFAULT 0,"
434 " key BLOB NOT NULL DEFAULT '',"
435 " value BLOB NOT NULL DEFAULT '',"
436 " path BLOB DEFAULT '')");
437 SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds090 (key,type,expire)");
438 SQLITE3_EXEC (dbh, "CREATE INDEX idx_expire ON ds090 (expire)");
439 plugin = GNUNET_malloc (sizeof (struct Plugin));
442 plugin->fn = fn_utf8;
443 api = GNUNET_malloc (sizeof (struct GNUNET_DATACACHE_PluginFunctions));
445 api->get = &sqlite_plugin_get;
446 api->put = &sqlite_plugin_put;
447 api->del = &sqlite_plugin_del;
448 LOG (GNUNET_ERROR_TYPE_INFO, _("Sqlite datacache running\n"));
454 * Exit point from the plugin.
456 * @param cls closure (our "struct Plugin")
460 libgnunet_plugin_datacache_sqlite_done (void *cls)
462 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
463 struct Plugin *plugin = api->cls;
466 #if SQLITE_VERSION_NUMBER >= 3007000
470 #if !WINDOWS || defined(__CYGWIN__)
471 if ( (NULL != plugin->fn) &&
472 (0 != UNLINK (plugin->fn)) )
473 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", plugin->fn);
474 GNUNET_free_non_null (plugin->fn);
476 result = sqlite3_close (plugin->dbh);
477 #if SQLITE_VERSION_NUMBER >= 3007000
478 if (result == SQLITE_BUSY)
480 LOG (GNUNET_ERROR_TYPE_WARNING,
482 ("Tried to close sqlite without finalizing all prepared statements.\n"));
483 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
486 LOG (GNUNET_ERROR_TYPE_DEBUG, "Closing statement %p\n", stmt);
487 result = sqlite3_finalize (stmt);
488 if (result != SQLITE_OK)
489 LOG (GNUNET_ERROR_TYPE_WARNING, _("Failed to close statement %p: %d\n"),
491 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
493 result = sqlite3_close (plugin->dbh);
496 if (SQLITE_OK != result)
497 LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
499 #if WINDOWS && !defined(__CYGWIN__)
500 if ( (NULL != plugin->fn) &&
501 (0 != UNLINK (plugin->fn)) )
502 LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", plugin->fn);
503 GNUNET_free_non_null (plugin->fn);
505 GNUNET_free (plugin);
512 /* end of plugin_datacache_sqlite.c */