2 This file is part of GNUnet
3 Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 * @file datacache/plugin_datacache_postgres.c
21 * @brief postgres for an implementation of a database backend for the datacache
22 * @author Christian Grothoff
25 #include "gnunet_util_lib.h"
26 #include "gnunet_pq_lib.h"
27 #include "gnunet_datacache_plugin.h"
29 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-postgres", __VA_ARGS__)
32 * Per-entry overhead estimate
34 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 24)
37 * Context for all functions in this plugin.
42 * Our execution environment.
44 struct GNUNET_DATACACHE_PluginEnvironment *env;
47 * Native Postgres database handle.
52 * Number of key-value pairs in the database.
54 unsigned int num_items;
59 * @brief Get a database handle
61 * @param plugin global context
62 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
65 init_connection (struct Plugin *plugin)
67 struct GNUNET_PQ_ExecuteStatement es[] = {
68 GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn011dc ("
69 " type INTEGER NOT NULL,"
70 " prox INTEGER NOT NULL,"
71 " discard_time BIGINT NOT NULL,"
72 " key BYTEA NOT NULL,"
73 " value BYTEA NOT NULL,"
74 " path BYTEA DEFAULT NULL)"
76 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_key ON gn011dc (key)"),
77 GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_dt ON gn011dc (discard_time)"),
78 GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER value SET STORAGE EXTERNAL"),
79 GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER key SET STORAGE PLAIN"),
80 GNUNET_PQ_EXECUTE_STATEMENT_END
82 struct GNUNET_PQ_PreparedStatement ps[] = {
83 GNUNET_PQ_make_prepare ("getkt",
84 "SELECT discard_time,type,value,path FROM gn011dc "
85 "WHERE key=$1 AND type=$2 AND discard_time >= $3",
87 GNUNET_PQ_make_prepare ("getk",
88 "SELECT discard_time,type,value,path FROM gn011dc "
89 "WHERE key=$1 AND discard_time >= $2",
91 GNUNET_PQ_make_prepare ("getex",
92 "SELECT length(value) AS len,oid,key FROM gn011dc"
93 " WHERE discard_time < $1"
94 " ORDER BY discard_time ASC LIMIT 1",
96 GNUNET_PQ_make_prepare ("getm",
97 "SELECT length(value) AS len,oid,key FROM gn011dc"
98 " ORDER BY prox ASC, discard_time ASC LIMIT 1",
100 GNUNET_PQ_make_prepare ("get_random",
101 "SELECT discard_time,type,value,path,key FROM gn011dc"
102 " WHERE discard_time >= $1"
103 " ORDER BY key ASC LIMIT 1 OFFSET $2",
105 GNUNET_PQ_make_prepare ("get_closest",
106 "SELECT discard_time,type,value,path,key FROM gn011dc "
107 "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3",
109 GNUNET_PQ_make_prepare ("delrow",
110 "DELETE FROM gn011dc WHERE oid=$1",
112 GNUNET_PQ_make_prepare ("put",
113 "INSERT INTO gn011dc (type, prox, discard_time, key, value, path) "
114 "VALUES ($1, $2, $3, $4, $5, $6)",
116 GNUNET_PQ_PREPARED_STATEMENT_END
119 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
120 "datacache-postgres");
121 if (NULL == plugin->dbh)
122 return GNUNET_SYSERR;
124 GNUNET_PQ_exec_statements (plugin->dbh,
127 PQfinish (plugin->dbh);
129 return GNUNET_SYSERR;
133 GNUNET_PQ_prepare_statements (plugin->dbh,
136 PQfinish (plugin->dbh);
138 return GNUNET_SYSERR;
145 * Store an item in the datastore.
147 * @param cls closure (our `struct Plugin`)
148 * @param key key to store @a data under
149 * @param prox proximity of @a key to my PID
150 * @param data_size number of bytes in @a data
151 * @param data data to store
152 * @param type type of the value
153 * @param discard_time when to discard the value in any case
154 * @param path_info_len number of entries in @a path_info
155 * @param path_info a path through the network
156 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
159 postgres_plugin_put (void *cls,
160 const struct GNUNET_HashCode *key,
164 enum GNUNET_BLOCK_Type type,
165 struct GNUNET_TIME_Absolute discard_time,
166 unsigned int path_info_len,
167 const struct GNUNET_PeerIdentity *path_info)
169 struct Plugin *plugin = cls;
170 uint32_t type32 = (uint32_t) type;
171 struct GNUNET_PQ_QueryParam params[] = {
172 GNUNET_PQ_query_param_uint32 (&type32),
173 GNUNET_PQ_query_param_uint32 (&prox),
174 GNUNET_PQ_query_param_absolute_time (&discard_time),
175 GNUNET_PQ_query_param_auto_from_type (key),
176 GNUNET_PQ_query_param_fixed_size (data, data_size),
177 GNUNET_PQ_query_param_fixed_size (path_info,
178 path_info_len * sizeof (struct GNUNET_PeerIdentity)),
179 GNUNET_PQ_query_param_end
181 enum GNUNET_DB_QueryStatus ret;
183 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
189 return data_size + OVERHEAD;
194 * Closure for #handle_results.
196 struct HandleResultContext
200 * Function to call on each result, may be NULL.
202 GNUNET_DATACACHE_Iterator iter;
205 * Closure for @e iter.
212 const struct GNUNET_HashCode *key;
217 * Function to be called with the results of a SELECT statement
218 * that has returned @a num_results results. Parse the result
219 * and call the callback given in @a cls
221 * @param cls closure of type `struct HandleResultContext`
222 * @param result the postgres result
223 * @param num_result the number of results in @a result
226 handle_results (void *cls,
228 unsigned int num_results)
230 struct HandleResultContext *hrc = cls;
232 for (unsigned int i=0;i<num_results;i++)
234 struct GNUNET_TIME_Absolute expiration_time;
238 struct GNUNET_PeerIdentity *path;
240 struct GNUNET_PQ_ResultSpec rs[] = {
241 GNUNET_PQ_result_spec_absolute_time ("discard_time",
243 GNUNET_PQ_result_spec_uint32 ("type",
245 GNUNET_PQ_result_spec_variable_size ("value",
248 GNUNET_PQ_result_spec_variable_size ("path",
251 GNUNET_PQ_result_spec_end
255 GNUNET_PQ_extract_result (result,
262 if (0 != (path_len % sizeof (struct GNUNET_PeerIdentity)))
267 path_len %= sizeof (struct GNUNET_PeerIdentity);
268 LOG (GNUNET_ERROR_TYPE_DEBUG,
269 "Found result of size %u bytes and type %u in database\n",
270 (unsigned int) data_size,
271 (unsigned int) type);
272 if ( (NULL != hrc->iter) &&
274 hrc->iter (hrc->iter_cls,
278 (enum GNUNET_BLOCK_Type) type,
283 LOG (GNUNET_ERROR_TYPE_DEBUG,
284 "Ending iteration (client error)\n");
285 GNUNET_PQ_cleanup_result (rs);
288 GNUNET_PQ_cleanup_result (rs);
294 * Iterate over the results for a particular key
297 * @param cls closure (our `struct Plugin`)
298 * @param key key to look for
299 * @param type entries of which type are relevant?
300 * @param iter maybe NULL (to just count)
301 * @param iter_cls closure for @a iter
302 * @return the number of results found
305 postgres_plugin_get (void *cls,
306 const struct GNUNET_HashCode *key,
307 enum GNUNET_BLOCK_Type type,
308 GNUNET_DATACACHE_Iterator iter,
311 struct Plugin *plugin = cls;
312 uint32_t type32 = (uint32_t) type;
313 struct GNUNET_TIME_Absolute now;
314 struct GNUNET_PQ_QueryParam paramk[] = {
315 GNUNET_PQ_query_param_auto_from_type (key),
316 GNUNET_PQ_query_param_absolute_time (&now),
317 GNUNET_PQ_query_param_end
319 struct GNUNET_PQ_QueryParam paramkt[] = {
320 GNUNET_PQ_query_param_auto_from_type (key),
321 GNUNET_PQ_query_param_uint32 (&type32),
322 GNUNET_PQ_query_param_absolute_time (&now),
323 GNUNET_PQ_query_param_end
325 enum GNUNET_DB_QueryStatus res;
326 struct HandleResultContext hr_ctx;
328 now = GNUNET_TIME_absolute_get ();
330 hr_ctx.iter_cls = iter_cls;
332 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
333 (0 == type) ? "getk" : "getkt",
334 (0 == type) ? paramk : paramkt,
344 * Delete the entry with the lowest expiration value
345 * from the datacache right now.
347 * @param cls closure (our `struct Plugin`)
348 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
351 postgres_plugin_del (void *cls)
353 struct Plugin *plugin = cls;
354 struct GNUNET_PQ_QueryParam pempty[] = {
355 GNUNET_PQ_query_param_end
359 struct GNUNET_HashCode key;
360 struct GNUNET_PQ_ResultSpec rs[] = {
361 GNUNET_PQ_result_spec_uint32 ("len",
363 GNUNET_PQ_result_spec_uint32 ("oid",
365 GNUNET_PQ_result_spec_auto_from_type ("key",
367 GNUNET_PQ_result_spec_end
369 enum GNUNET_DB_QueryStatus res;
370 struct GNUNET_PQ_QueryParam dparam[] = {
371 GNUNET_PQ_query_param_uint32 (&oid),
372 GNUNET_PQ_query_param_end
374 struct GNUNET_TIME_Absolute now;
375 struct GNUNET_PQ_QueryParam xparam[] = {
376 GNUNET_PQ_query_param_absolute_time (&now),
377 GNUNET_PQ_query_param_end
380 now = GNUNET_TIME_absolute_get ();
381 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
386 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
391 return GNUNET_SYSERR;
392 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
395 LOG (GNUNET_ERROR_TYPE_DEBUG,
396 "Ending iteration (no more results)\n");
399 res = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
404 GNUNET_PQ_cleanup_result (rs);
405 return GNUNET_SYSERR;
408 plugin->env->delete_notify (plugin->env->cls,
411 GNUNET_PQ_cleanup_result (rs);
417 * Obtain a random key-value pair from the datacache.
419 * @param cls closure (our `struct Plugin`)
420 * @param iter maybe NULL (to just count)
421 * @param iter_cls closure for @a iter
422 * @return the number of results found, zero (datacache empty) or one
425 postgres_plugin_get_random (void *cls,
426 GNUNET_DATACACHE_Iterator iter,
429 struct Plugin *plugin = cls;
431 struct GNUNET_TIME_Absolute now;
432 struct GNUNET_TIME_Absolute expiration_time;
436 struct GNUNET_PeerIdentity *path;
437 struct GNUNET_HashCode key;
439 enum GNUNET_DB_QueryStatus res;
440 struct GNUNET_PQ_QueryParam params[] = {
441 GNUNET_PQ_query_param_absolute_time (&now),
442 GNUNET_PQ_query_param_uint32 (&off),
443 GNUNET_PQ_query_param_end
445 struct GNUNET_PQ_ResultSpec rs[] = {
446 GNUNET_PQ_result_spec_absolute_time ("discard_time",
448 GNUNET_PQ_result_spec_uint32 ("type",
450 GNUNET_PQ_result_spec_variable_size ("value",
453 GNUNET_PQ_result_spec_variable_size ("path",
456 GNUNET_PQ_result_spec_auto_from_type ("key",
458 GNUNET_PQ_result_spec_end
461 if (0 == plugin->num_items)
465 now = GNUNET_TIME_absolute_get ();
466 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
468 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
477 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
482 if (0 != (path_len % sizeof (struct GNUNET_PeerIdentity)))
487 path_len %= sizeof (struct GNUNET_PeerIdentity);
488 LOG (GNUNET_ERROR_TYPE_DEBUG,
489 "Found random value with key %s of size %u bytes and type %u in database\n",
491 (unsigned int) data_size,
492 (unsigned int) type);
493 (void) iter (iter_cls,
497 (enum GNUNET_BLOCK_Type) type,
501 GNUNET_PQ_cleanup_result (rs);
507 * Closure for #extract_result_cb.
509 struct ExtractResultContext
512 * Function to call for each result found.
514 GNUNET_DATACACHE_Iterator iter;
517 * Closure for @e iter.
525 * Function to be called with the results of a SELECT statement
526 * that has returned @a num_results results. Calls the `iter`
527 * from @a cls for each result.
529 * @param cls closure with the `struct ExtractResultContext`
530 * @param result the postgres result
531 * @param num_result the number of results in @a result
534 extract_result_cb (void *cls,
536 unsigned int num_results)
538 struct ExtractResultContext *erc = cls;
540 if (NULL == erc->iter)
542 for (unsigned int i=0;i<num_results;i++)
544 struct GNUNET_TIME_Absolute expiration_time;
548 struct GNUNET_PeerIdentity *path;
550 struct GNUNET_HashCode key;
551 struct GNUNET_PQ_ResultSpec rs[] = {
552 GNUNET_PQ_result_spec_absolute_time ("",
554 GNUNET_PQ_result_spec_uint32 ("type",
556 GNUNET_PQ_result_spec_variable_size ("value",
559 GNUNET_PQ_result_spec_variable_size ("path",
562 GNUNET_PQ_result_spec_auto_from_type ("key",
564 GNUNET_PQ_result_spec_end
568 GNUNET_PQ_extract_result (result,
575 if (0 != (path_len % sizeof (struct GNUNET_PeerIdentity)))
580 path_len %= sizeof (struct GNUNET_PeerIdentity);
581 LOG (GNUNET_ERROR_TYPE_DEBUG,
582 "Found result of size %u bytes and type %u in database\n",
583 (unsigned int) data_size,
584 (unsigned int) type);
586 erc->iter (erc->iter_cls,
590 (enum GNUNET_BLOCK_Type) type,
595 LOG (GNUNET_ERROR_TYPE_DEBUG,
596 "Ending iteration (client error)\n");
597 GNUNET_PQ_cleanup_result (rs);
600 GNUNET_PQ_cleanup_result (rs);
606 * Iterate over the results that are "close" to a particular key in
607 * the datacache. "close" is defined as numerically larger than @a
608 * key (when interpreted as a circular address space), with small
611 * @param cls closure (internal context for the plugin)
612 * @param key area of the keyspace to look into
613 * @param num_results number of results that should be returned to @a iter
614 * @param iter maybe NULL (to just count)
615 * @param iter_cls closure for @a iter
616 * @return the number of results found
619 postgres_plugin_get_closest (void *cls,
620 const struct GNUNET_HashCode *key,
621 unsigned int num_results,
622 GNUNET_DATACACHE_Iterator iter,
625 struct Plugin *plugin = cls;
626 uint32_t num_results32 = (uint32_t) num_results;
627 struct GNUNET_TIME_Absolute now;
628 struct GNUNET_PQ_QueryParam params[] = {
629 GNUNET_PQ_query_param_auto_from_type (key),
630 GNUNET_PQ_query_param_absolute_time (&now),
631 GNUNET_PQ_query_param_uint32 (&num_results32),
632 GNUNET_PQ_query_param_end
634 enum GNUNET_DB_QueryStatus res;
635 struct ExtractResultContext erc;
638 erc.iter_cls = iter_cls;
639 now = GNUNET_TIME_absolute_get ();
640 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
647 LOG (GNUNET_ERROR_TYPE_DEBUG,
648 "Ending iteration (postgres error)\n");
651 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
654 LOG (GNUNET_ERROR_TYPE_DEBUG,
655 "Ending iteration (no more results)\n");
663 * Entry point for the plugin.
665 * @param cls closure (the `struct GNUNET_DATACACHE_PluginEnvironmnet`)
666 * @return the plugin's closure (our `struct Plugin`)
669 libgnunet_plugin_datacache_postgres_init (void *cls)
671 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
672 struct GNUNET_DATACACHE_PluginFunctions *api;
673 struct Plugin *plugin;
675 plugin = GNUNET_new (struct Plugin);
678 if (GNUNET_OK != init_connection (plugin))
680 GNUNET_free (plugin);
684 api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
686 api->get = &postgres_plugin_get;
687 api->put = &postgres_plugin_put;
688 api->del = &postgres_plugin_del;
689 api->get_random = &postgres_plugin_get_random;
690 api->get_closest = &postgres_plugin_get_closest;
691 LOG (GNUNET_ERROR_TYPE_INFO,
692 "Postgres datacache running\n");
698 * Exit point from the plugin.
700 * @param cls closure (our `struct Plugin`)
704 libgnunet_plugin_datacache_postgres_done (void *cls)
706 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
707 struct Plugin *plugin = api->cls;
709 PQfinish (plugin->dbh);
710 GNUNET_free (plugin);
717 /* end of plugin_datacache_postgres.c */