2 This file is part of GNUnet.
3 (C) 2006, 2007, 2008 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file applications/dstore_sqlite/dstore.c
23 * @brief SQLite based implementation of the dstore service
24 * @author Christian Grothoff
25 * @todo Indexes, statistics
31 #include "gnunet_util.h"
32 #include "gnunet_dstore_service.h"
33 #include "gnunet_stats_service.h"
36 #define DEBUG_DSTORE GNUNET_NO
39 * Maximum size for an individual item.
41 #define MAX_CONTENT_SIZE 65536
46 static unsigned long long payload;
49 * Maximum bytes available
51 static unsigned long long quota;
54 * Filename of this database
59 static GNUNET_CoreAPIForPlugins *coreAPI;
61 static struct GNUNET_Mutex *lock;
66 static GNUNET_Stats_ServiceAPI *stats;
68 static unsigned int stat_dstore_size;
70 static unsigned int stat_dstore_quota;
73 * Estimate of the per-entry overhead (including indices).
75 #define OVERHEAD ((4*2+4*2+8*2+8*2+sizeof(GNUNET_HashCode)*5+32))
77 struct GNUNET_BloomFilter *bloom;
79 static char *bloom_name;
82 * @brief Prepare a SQL statement
85 sq_prepare (sqlite3 * dbh, const char *zSql, /* SQL statement, UTF-8 encoded */
86 sqlite3_stmt ** ppStmt)
87 { /* OUT: Statement handle */
89 return sqlite3_prepare (dbh,
91 strlen (zSql), ppStmt, (const char **) &dummy);
94 #define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { GNUNET_GE_LOG(coreAPI->ectx, GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0)
97 * Log an error message at log-level 'level' that indicates
98 * a failure of the command 'cmd' on file 'filename'
99 * with the message given by strerror(errno).
101 #define LOG_SQLITE(db, level, cmd) do { GNUNET_GE_LOG(coreAPI->ectx, level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0)
104 db_init (sqlite3 * dbh)
108 SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY");
109 SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF");
110 SQLITE3_EXEC (dbh, "PRAGMA count_changes=OFF");
111 SQLITE3_EXEC (dbh, "PRAGMA page_size=4092");
113 "CREATE TABLE ds080 ("
114 " size INTEGER NOT NULL DEFAULT 0,"
115 " type INTEGER NOT NULL DEFAULT 0,"
116 " puttime INTEGER NOT NULL DEFAULT 0,"
117 " expire INTEGER NOT NULL DEFAULT 0,"
118 " key BLOB NOT NULL DEFAULT '',"
119 " vhash BLOB NOT NULL DEFAULT '',"
120 " value BLOB NOT NULL DEFAULT '')");
121 SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds080 (key,type,expire)");
123 "CREATE INDEX idx_allidx ON ds080 (key,vhash,type,size)");
124 SQLITE3_EXEC (dbh, "CREATE INDEX idx_puttime ON ds080 (puttime)");
139 GNUNET_free (fn_utf8);
143 tmpdir = getenv ("TMPDIR");
144 tmpdir = tmpdir ? tmpdir : "/tmp";
146 #define TEMPLATE "/gnunet-dstoreXXXXXX"
147 tmpl = GNUNET_malloc (strlen (tmpdir) + sizeof (TEMPLATE) + 1);
148 strcpy (tmpl, tmpdir);
149 strcat (tmpl, TEMPLATE);
153 fn = (char *) GNUNET_malloc (MAX_PATH + 1);
154 plibc_conv_to_win_path (tmpl, fn);
162 GNUNET_GE_BREAK (NULL, 0);
165 return GNUNET_SYSERR;
168 fn_utf8 = GNUNET_convert_string_to_utf8 (coreAPI->ectx, fn, strlen (fn),
170 nl_langinfo (CODESET)
172 "UTF-8" /* good luck */
175 if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))
178 GNUNET_free (fn_utf8);
180 return GNUNET_SYSERR;
188 * Check that we are within quota.
189 * @return GNUNET_OK if we are.
192 checkQuota (sqlite3 * dbh)
194 GNUNET_HashCode dkey;
195 GNUNET_HashCode vhash;
202 if (payload * 10 <= quota * 9)
203 return GNUNET_OK; /* we seem to be about 10% off */
205 GNUNET_GE_LOG (coreAPI->ectx,
206 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER,
207 "DStore above qutoa (have %llu, allowed %llu), will delete some data.\n",
212 if ((sq_prepare (dbh,
213 "SELECT size, type, key, vhash FROM ds080 ORDER BY puttime ASC LIMIT 1",
214 &stmt) != SQLITE_OK) ||
217 "WHERE key=? AND vhash=? AND type=? AND size=?",
218 &dstmt) != SQLITE_OK))
220 GNUNET_GE_LOG (coreAPI->ectx,
221 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
222 _("`%s' failed at %s:%d with error: %s\n"),
223 "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
224 GNUNET_GE_BREAK (NULL, 0);
226 sqlite3_finalize (dstmt);
228 sqlite3_finalize (stmt);
229 return GNUNET_SYSERR;
232 while ((payload * 10 > quota * 9) && /* we seem to be about 10% off */
233 ((err = sqlite3_step (stmt)) == SQLITE_ROW))
235 dsize = sqlite3_column_int (stmt, 0);
236 dtype = sqlite3_column_int (stmt, 1);
237 GNUNET_GE_BREAK (NULL,
238 sqlite3_column_bytes (stmt,
239 2) == sizeof (GNUNET_HashCode));
240 GNUNET_GE_BREAK (NULL,
241 sqlite3_column_bytes (stmt,
242 3) == sizeof (GNUNET_HashCode));
243 memcpy (&dkey, sqlite3_column_blob (stmt, 2), sizeof (GNUNET_HashCode));
244 memcpy (&vhash, sqlite3_column_blob (stmt, 3),
245 sizeof (GNUNET_HashCode));
246 sqlite3_reset (stmt);
247 sqlite3_bind_blob (dstmt,
248 1, &dkey, sizeof (GNUNET_HashCode),
250 sqlite3_bind_blob (dstmt,
251 2, &vhash, sizeof (GNUNET_HashCode),
253 sqlite3_bind_int (dstmt, 3, dtype);
254 sqlite3_bind_int (dstmt, 4, dsize);
255 if ((err = sqlite3_step (dstmt)) != SQLITE_DONE)
257 GNUNET_GE_LOG (coreAPI->ectx,
258 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
259 _("`%s' failed at %s:%d with error: %s\n"),
260 "sqlite3_step", __FILE__, __LINE__,
261 sqlite3_errmsg (dbh));
262 sqlite3_reset (dstmt);
263 GNUNET_GE_BREAK (NULL, 0); /* should delete but cannot!? */
266 if (sqlite3_total_changes (dbh) > 0)
269 GNUNET_bloomfilter_remove (bloom, &dkey);
270 payload -= (dsize + OVERHEAD);
273 GNUNET_GE_LOG (coreAPI->ectx,
274 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST |
276 "Deleting %u bytes decreases DStore payload to %llu out of %llu\n",
277 dsize, payload, quota);
279 sqlite3_reset (dstmt);
281 if (err != SQLITE_DONE)
283 GNUNET_GE_LOG (coreAPI->ectx,
284 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
285 _("`%s' failed at %s:%d with error: %s\n"),
286 "sqlite3_step", __FILE__, __LINE__,
287 sqlite3_errmsg (dbh));
289 sqlite3_finalize (dstmt);
290 sqlite3_finalize (stmt);
291 if (payload * 10 > quota * 9)
293 /* we seem to be about 10% off */
294 GNUNET_GE_LOG (coreAPI->ectx,
295 GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_DEVELOPER,
296 "Failed to delete content to drop below quota (bug?).\n",
298 return GNUNET_SYSERR;
304 * Store an item in the datastore.
306 * @return GNUNET_OK on success, GNUNET_SYSERR on error
309 d_put (const GNUNET_HashCode * key,
311 GNUNET_CronTime discard_time, unsigned int size, const char *data)
313 GNUNET_HashCode vhash;
319 if (size > MAX_CONTENT_SIZE)
320 return GNUNET_SYSERR;
321 GNUNET_hash (data, size, &vhash);
322 GNUNET_mutex_lock (lock);
323 if ((fn == NULL) || (SQLITE_OK != sqlite3_open (fn_utf8, &dbh)))
326 GNUNET_mutex_unlock (lock);
327 return GNUNET_SYSERR;
329 now = GNUNET_get_time ();
331 GNUNET_GE_LOG (coreAPI->ectx,
332 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER,
333 "dstore processes put `%.*s' with expiration %llu\n",
334 size, data, discard_time);
337 /* first try UPDATE */
339 "UPDATE ds080 SET puttime=?, expire=? "
340 "WHERE key=? AND vhash=? AND type=? AND size=?",
343 GNUNET_GE_LOG (coreAPI->ectx,
344 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
345 _("`%s' failed at %s:%d with error: %s\n"),
346 "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
348 GNUNET_mutex_unlock (lock);
349 return GNUNET_SYSERR;
352 sqlite3_bind_int64 (stmt, 1, now)) ||
354 sqlite3_bind_int64 (stmt, 2, discard_time)) ||
356 sqlite3_bind_blob (stmt, 3, key, sizeof (GNUNET_HashCode),
357 SQLITE_TRANSIENT)) ||
359 sqlite3_bind_blob (stmt, 4, &vhash, sizeof (GNUNET_HashCode),
360 SQLITE_TRANSIENT)) ||
361 (SQLITE_OK != sqlite3_bind_int (stmt, 5, type)) ||
362 (SQLITE_OK != sqlite3_bind_int (stmt, 6, size)))
364 GNUNET_GE_LOG (coreAPI->ectx,
365 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
366 _("`%s' failed at %s:%d with error: %s\n"),
367 "sqlite3_bind_xxx", __FILE__, __LINE__,
368 sqlite3_errmsg (dbh));
369 sqlite3_finalize (stmt);
371 GNUNET_mutex_unlock (lock);
372 return GNUNET_SYSERR;
374 if (SQLITE_DONE != sqlite3_step (stmt))
376 GNUNET_GE_LOG (coreAPI->ectx,
377 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
378 _("`%s' failed at %s:%d with error: %s\n"),
379 "sqlite3_step", __FILE__, __LINE__,
380 sqlite3_errmsg (dbh));
381 sqlite3_finalize (stmt);
383 GNUNET_mutex_unlock (lock);
384 return GNUNET_SYSERR;
386 ret = sqlite3_changes (dbh);
387 sqlite3_finalize (stmt);
391 GNUNET_mutex_unlock (lock);
394 if (GNUNET_OK != checkQuota (dbh))
397 GNUNET_mutex_unlock (lock);
398 return GNUNET_SYSERR;
402 "(size, type, puttime, expire, key, vhash, value) "
403 "VALUES (?, ?, ?, ?, ?, ?, ?)", &stmt) != SQLITE_OK)
405 GNUNET_GE_LOG (coreAPI->ectx,
406 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
407 _("`%s' failed at %s:%d with error: %s\n"),
408 "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
410 GNUNET_mutex_unlock (lock);
411 return GNUNET_SYSERR;
413 if ((SQLITE_OK == sqlite3_bind_int (stmt, 1, size)) &&
414 (SQLITE_OK == sqlite3_bind_int (stmt, 2, type)) &&
415 (SQLITE_OK == sqlite3_bind_int64 (stmt, 3, now)) &&
416 (SQLITE_OK == sqlite3_bind_int64 (stmt, 4, discard_time)) &&
418 sqlite3_bind_blob (stmt, 5, key, sizeof (GNUNET_HashCode),
419 SQLITE_TRANSIENT)) &&
421 sqlite3_bind_blob (stmt, 6, &vhash, sizeof (GNUNET_HashCode),
424 sqlite3_bind_blob (stmt, 7, data, size, SQLITE_TRANSIENT)))
426 if (SQLITE_DONE != sqlite3_step (stmt))
429 GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN
430 | GNUNET_GE_BULK, "sqlite3_step");
434 payload += size + OVERHEAD;
436 GNUNET_bloomfilter_add (bloom, key);
438 if (SQLITE_OK != sqlite3_finalize (stmt))
440 GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN |
441 GNUNET_GE_BULK, "sqlite3_finalize");
446 GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN |
447 GNUNET_GE_BULK, "sqlite3_bind_xxx");
450 GNUNET_GE_LOG (coreAPI->ectx,
451 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER,
452 "Storing %u bytes increases DStore payload to %llu out of %llu\n",
453 size, payload, quota);
457 GNUNET_mutex_unlock (lock);
459 stats->set (stat_dstore_size, payload);
464 * Iterate over the results for a particular key
468 * @param type entries of which type are relevant?
469 * @param iter maybe NULL (to just count)
470 * @return the number of results, GNUNET_SYSERR if the
471 * iter is non-NULL and aborted the iteration
474 d_get (const GNUNET_HashCode * key,
475 unsigned int type, GNUNET_ResultProcessor handler, void *closure)
487 GNUNET_mutex_lock (lock);
488 if ((bloom != NULL) && (GNUNET_NO == GNUNET_bloomfilter_test (bloom, key)))
490 GNUNET_mutex_unlock (lock);
493 if ((fn == NULL) || (SQLITE_OK != sqlite3_open (fn_utf8, &dbh)))
496 GNUNET_mutex_unlock (lock);
497 return GNUNET_SYSERR;
499 now = GNUNET_get_time ();
501 GNUNET_GE_LOG (coreAPI->ectx,
502 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER,
503 "dstore processes get at `%llu'\n", now);
506 "SELECT count(*) FROM ds080 WHERE key=? AND type=? AND expire >= ?",
509 GNUNET_GE_LOG (coreAPI->ectx,
510 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
511 _("`%s' failed at %s:%d with error: %s\n"),
512 "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh));
515 GNUNET_mutex_unlock (lock);
516 return GNUNET_SYSERR;
518 sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode),
520 sqlite3_bind_int (stmt, 2, type);
521 sqlite3_bind_int64 (stmt, 3, now);
522 if (SQLITE_ROW != sqlite3_step (stmt))
525 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
526 GNUNET_GE_BULK, "sqlite_step");
527 sqlite3_reset (stmt);
528 sqlite3_finalize (stmt);
530 GNUNET_mutex_unlock (lock);
531 return GNUNET_SYSERR;
533 total = sqlite3_column_int (stmt, 0);
534 sqlite3_reset (stmt);
535 sqlite3_finalize (stmt);
536 if ((total == 0) || (handler == NULL))
539 GNUNET_mutex_unlock (lock);
544 off = GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, total);
547 off = (off + 1) % total;
548 GNUNET_snprintf (scratch, 256,
549 "SELECT size, value FROM ds080 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
551 if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK)
553 GNUNET_GE_LOG (coreAPI->ectx,
554 GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK,
555 _("`%s' failed at %s:%d with error: %s\n"),
556 "sq_prepare", __FILE__, __LINE__,
557 sqlite3_errmsg (dbh));
559 GNUNET_mutex_unlock (lock);
560 return GNUNET_SYSERR;
562 sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode),
564 sqlite3_bind_int (stmt, 2, type);
565 sqlite3_bind_int64 (stmt, 3, now);
566 if (sqlite3_step (stmt) != SQLITE_ROW)
568 size = sqlite3_column_int (stmt, 0);
569 if (size != sqlite3_column_bytes (stmt, 1))
571 GNUNET_GE_BREAK (NULL, 0);
572 sqlite3_finalize (stmt);
575 dat = sqlite3_column_blob (stmt, 1);
578 GNUNET_GE_LOG (coreAPI->ectx,
579 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST |
581 "dstore found result for get: `%.*s'\n", size, dat);
583 if ((handler != NULL) &&
584 (GNUNET_OK != handler (key, type, size, dat, closure)))
586 sqlite3_finalize (stmt);
589 sqlite3_finalize (stmt);
592 GNUNET_mutex_unlock (lock);
596 GNUNET_Dstore_ServiceAPI *
597 provide_module_dstore_sqlite (GNUNET_CoreAPIForPlugins * capi)
599 static GNUNET_Dstore_ServiceAPI api;
603 GNUNET_GE_LOG (capi->ectx,
604 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
605 "SQLite Dstore: initializing database\n");
608 if (GNUNET_OK != db_reset ())
610 GNUNET_GE_BREAK (capi->ectx, 0);
613 lock = GNUNET_mutex_create (GNUNET_NO);
618 GNUNET_GC_get_configuration_value_number (coreAPI->cfg,
619 "DSTORE", "QUOTA", 1, 1024, 1,
621 if (quota == 0) /* error */
623 quota *= 1024 * 1024;
625 bloom_name = GNUNET_strdup ("/tmp/dbloomXXXXXX");
626 fd = mkstemp (bloom_name);
629 bloom = GNUNET_bloomfilter_load (coreAPI->ectx, bloom_name, quota / (OVERHEAD + 1024), /* 8 bit per entry in DB, expect 1k entries */
633 stats = capi->service_request ("stats");
636 stat_dstore_size = stats->create (gettext_noop ("# bytes in dstore"));
638 stats->create (gettext_noop ("# max bytes allowed in dstore"));
639 stats->set (stat_dstore_quota, quota);
645 * Shutdown the module.
648 release_module_dstore_sqlite ()
652 GNUNET_free (fn_utf8);
656 GNUNET_bloomfilter_free (bloom);
660 GNUNET_free (bloom_name);
664 coreAPI->service_release (stats);
668 GNUNET_GE_LOG (coreAPI->ectx,
669 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
670 "SQLite Dstore: database shutdown\n");
672 GNUNET_mutex_destroy (lock);
676 /* end of dstore.c */