2 This file is part of GNUnet
3 Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @file datacache/plugin_datacache_postgres.c
23 * @brief postgres for an implementation of a database backend for the datacache
24 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_pq_lib.h"
29 #include "gnunet_datacache_plugin.h"
31 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache-postgres", __VA_ARGS__)
34 * Per-entry overhead estimate
36 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 24)
39 * Context for all functions in this plugin.
44 * Our execution environment.
46 struct GNUNET_DATACACHE_PluginEnvironment *env;
49 * Native Postgres database handle.
51 struct GNUNET_PQ_Context *dbh;
54 * Number of key-value pairs in the database.
56 unsigned int num_items;
61 * @brief Get a database handle
63 * @param plugin global context
64 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
67 init_connection (struct Plugin *plugin)
69 struct GNUNET_PQ_ExecuteStatement es[] = {
70 GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn011dc ("
71 " type INTEGER NOT NULL,"
72 " prox INTEGER NOT NULL,"
73 " discard_time BIGINT NOT NULL,"
74 " key BYTEA NOT NULL,"
75 " value BYTEA NOT NULL,"
76 " path BYTEA DEFAULT NULL)"
78 GNUNET_PQ_make_try_execute (
79 "CREATE INDEX IF NOT EXISTS idx_key ON gn011dc (key)"),
80 GNUNET_PQ_make_try_execute (
81 "CREATE INDEX IF NOT EXISTS idx_dt ON gn011dc (discard_time)"),
82 GNUNET_PQ_make_execute (
83 "ALTER TABLE gn011dc ALTER value SET STORAGE EXTERNAL"),
84 GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER key SET STORAGE PLAIN"),
85 GNUNET_PQ_EXECUTE_STATEMENT_END
87 struct GNUNET_PQ_PreparedStatement ps[] = {
88 GNUNET_PQ_make_prepare ("getkt",
89 "SELECT discard_time,type,value,path FROM gn011dc "
90 "WHERE key=$1 AND type=$2 AND discard_time >= $3",
92 GNUNET_PQ_make_prepare ("getk",
93 "SELECT discard_time,type,value,path FROM gn011dc "
94 "WHERE key=$1 AND discard_time >= $2",
96 GNUNET_PQ_make_prepare ("getex",
97 "SELECT length(value) AS len,oid,key FROM gn011dc"
98 " WHERE discard_time < $1"
99 " ORDER BY discard_time ASC LIMIT 1",
101 GNUNET_PQ_make_prepare ("getm",
102 "SELECT length(value) AS len,oid,key FROM gn011dc"
103 " ORDER BY prox ASC, discard_time ASC LIMIT 1",
105 GNUNET_PQ_make_prepare ("get_random",
106 "SELECT discard_time,type,value,path,key FROM gn011dc"
107 " WHERE discard_time >= $1"
108 " ORDER BY key ASC LIMIT 1 OFFSET $2",
110 GNUNET_PQ_make_prepare ("get_closest",
111 "SELECT discard_time,type,value,path,key FROM gn011dc "
112 "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3",
114 GNUNET_PQ_make_prepare ("delrow",
115 "DELETE FROM gn011dc WHERE oid=$1",
117 GNUNET_PQ_make_prepare ("put",
118 "INSERT INTO gn011dc (type, prox, discard_time, key, value, path) "
119 "VALUES ($1, $2, $3, $4, $5, $6)",
121 GNUNET_PQ_PREPARED_STATEMENT_END
124 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
125 "datacache-postgres",
129 if (NULL == plugin->dbh)
130 return GNUNET_SYSERR;
136 * Store an item in the datastore.
138 * @param cls closure (our `struct Plugin`)
139 * @param key key to store @a data under
140 * @param prox proximity of @a key to my PID
141 * @param data_size number of bytes in @a data
142 * @param data data to store
143 * @param type type of the value
144 * @param discard_time when to discard the value in any case
145 * @param path_info_len number of entries in @a path_info
146 * @param path_info a path through the network
147 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
150 postgres_plugin_put (void *cls,
151 const struct GNUNET_HashCode *key,
155 enum GNUNET_BLOCK_Type type,
156 struct GNUNET_TIME_Absolute discard_time,
157 unsigned int path_info_len,
158 const struct GNUNET_PeerIdentity *path_info)
160 struct Plugin *plugin = cls;
161 uint32_t type32 = (uint32_t) type;
162 struct GNUNET_PQ_QueryParam params[] = {
163 GNUNET_PQ_query_param_uint32 (&type32),
164 GNUNET_PQ_query_param_uint32 (&prox),
165 GNUNET_PQ_query_param_absolute_time (&discard_time),
166 GNUNET_PQ_query_param_auto_from_type (key),
167 GNUNET_PQ_query_param_fixed_size (data, data_size),
168 GNUNET_PQ_query_param_fixed_size (path_info,
169 path_info_len * sizeof(struct
170 GNUNET_PeerIdentity)),
171 GNUNET_PQ_query_param_end
173 enum GNUNET_DB_QueryStatus ret;
175 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
181 return data_size + OVERHEAD;
186 * Closure for #handle_results.
188 struct HandleResultContext
191 * Function to call on each result, may be NULL.
193 GNUNET_DATACACHE_Iterator iter;
196 * Closure for @e iter.
203 const struct GNUNET_HashCode *key;
208 * Function to be called with the results of a SELECT statement
209 * that has returned @a num_results results. Parse the result
210 * and call the callback given in @a cls
212 * @param cls closure of type `struct HandleResultContext`
213 * @param result the postgres result
214 * @param num_result the number of results in @a result
217 handle_results (void *cls,
219 unsigned int num_results)
221 struct HandleResultContext *hrc = cls;
223 for (unsigned int i = 0; i < num_results; i++)
225 struct GNUNET_TIME_Absolute expiration_time;
229 struct GNUNET_PeerIdentity *path;
231 struct GNUNET_PQ_ResultSpec rs[] = {
232 GNUNET_PQ_result_spec_absolute_time ("discard_time",
234 GNUNET_PQ_result_spec_uint32 ("type",
236 GNUNET_PQ_result_spec_variable_size ("value",
239 GNUNET_PQ_result_spec_variable_size ("path",
242 GNUNET_PQ_result_spec_end
246 GNUNET_PQ_extract_result (result,
253 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
258 path_len %= sizeof(struct GNUNET_PeerIdentity);
259 LOG (GNUNET_ERROR_TYPE_DEBUG,
260 "Found result of size %u bytes and type %u in database\n",
261 (unsigned int) data_size,
262 (unsigned int) type);
263 if ((NULL != hrc->iter) &&
265 hrc->iter (hrc->iter_cls,
269 (enum GNUNET_BLOCK_Type) type,
274 LOG (GNUNET_ERROR_TYPE_DEBUG,
275 "Ending iteration (client error)\n");
276 GNUNET_PQ_cleanup_result (rs);
279 GNUNET_PQ_cleanup_result (rs);
285 * Iterate over the results for a particular key
288 * @param cls closure (our `struct Plugin`)
289 * @param key key to look for
290 * @param type entries of which type are relevant?
291 * @param iter maybe NULL (to just count)
292 * @param iter_cls closure for @a iter
293 * @return the number of results found
296 postgres_plugin_get (void *cls,
297 const struct GNUNET_HashCode *key,
298 enum GNUNET_BLOCK_Type type,
299 GNUNET_DATACACHE_Iterator iter,
302 struct Plugin *plugin = cls;
303 uint32_t type32 = (uint32_t) type;
304 struct GNUNET_TIME_Absolute now;
305 struct GNUNET_PQ_QueryParam paramk[] = {
306 GNUNET_PQ_query_param_auto_from_type (key),
307 GNUNET_PQ_query_param_absolute_time (&now),
308 GNUNET_PQ_query_param_end
310 struct GNUNET_PQ_QueryParam paramkt[] = {
311 GNUNET_PQ_query_param_auto_from_type (key),
312 GNUNET_PQ_query_param_uint32 (&type32),
313 GNUNET_PQ_query_param_absolute_time (&now),
314 GNUNET_PQ_query_param_end
316 enum GNUNET_DB_QueryStatus res;
317 struct HandleResultContext hr_ctx;
319 now = GNUNET_TIME_absolute_get ();
321 hr_ctx.iter_cls = iter_cls;
323 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
324 (0 == type) ? "getk" : "getkt",
325 (0 == type) ? paramk : paramkt,
335 * Delete the entry with the lowest expiration value
336 * from the datacache right now.
338 * @param cls closure (our `struct Plugin`)
339 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
342 postgres_plugin_del (void *cls)
344 struct Plugin *plugin = cls;
345 struct GNUNET_PQ_QueryParam pempty[] = {
346 GNUNET_PQ_query_param_end
350 struct GNUNET_HashCode key;
351 struct GNUNET_PQ_ResultSpec rs[] = {
352 GNUNET_PQ_result_spec_uint32 ("len",
354 GNUNET_PQ_result_spec_uint32 ("oid",
356 GNUNET_PQ_result_spec_auto_from_type ("key",
358 GNUNET_PQ_result_spec_end
360 enum GNUNET_DB_QueryStatus res;
361 struct GNUNET_PQ_QueryParam dparam[] = {
362 GNUNET_PQ_query_param_uint32 (&oid),
363 GNUNET_PQ_query_param_end
365 struct GNUNET_TIME_Absolute now;
366 struct GNUNET_PQ_QueryParam xparam[] = {
367 GNUNET_PQ_query_param_absolute_time (&now),
368 GNUNET_PQ_query_param_end
371 now = GNUNET_TIME_absolute_get ();
372 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
377 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
382 return GNUNET_SYSERR;
383 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
386 LOG (GNUNET_ERROR_TYPE_DEBUG,
387 "Ending iteration (no more results)\n");
390 res = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
395 GNUNET_PQ_cleanup_result (rs);
396 return GNUNET_SYSERR;
399 plugin->env->delete_notify (plugin->env->cls,
402 GNUNET_PQ_cleanup_result (rs);
408 * Obtain a random key-value pair from the datacache.
410 * @param cls closure (our `struct Plugin`)
411 * @param iter maybe NULL (to just count)
412 * @param iter_cls closure for @a iter
413 * @return the number of results found, zero (datacache empty) or one
416 postgres_plugin_get_random (void *cls,
417 GNUNET_DATACACHE_Iterator iter,
420 struct Plugin *plugin = cls;
422 struct GNUNET_TIME_Absolute now;
423 struct GNUNET_TIME_Absolute expiration_time;
427 struct GNUNET_PeerIdentity *path;
428 struct GNUNET_HashCode key;
430 enum GNUNET_DB_QueryStatus res;
431 struct GNUNET_PQ_QueryParam params[] = {
432 GNUNET_PQ_query_param_absolute_time (&now),
433 GNUNET_PQ_query_param_uint32 (&off),
434 GNUNET_PQ_query_param_end
436 struct GNUNET_PQ_ResultSpec rs[] = {
437 GNUNET_PQ_result_spec_absolute_time ("discard_time",
439 GNUNET_PQ_result_spec_uint32 ("type",
441 GNUNET_PQ_result_spec_variable_size ("value",
444 GNUNET_PQ_result_spec_variable_size ("path",
447 GNUNET_PQ_result_spec_auto_from_type ("key",
449 GNUNET_PQ_result_spec_end
452 if (0 == plugin->num_items)
456 now = GNUNET_TIME_absolute_get ();
457 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
459 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
468 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
473 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
478 path_len %= sizeof(struct GNUNET_PeerIdentity);
479 LOG (GNUNET_ERROR_TYPE_DEBUG,
480 "Found random value with key %s of size %u bytes and type %u in database\n",
482 (unsigned int) data_size,
483 (unsigned int) type);
484 (void) iter (iter_cls,
488 (enum GNUNET_BLOCK_Type) type,
492 GNUNET_PQ_cleanup_result (rs);
498 * Closure for #extract_result_cb.
500 struct ExtractResultContext
503 * Function to call for each result found.
505 GNUNET_DATACACHE_Iterator iter;
508 * Closure for @e iter.
515 * Function to be called with the results of a SELECT statement
516 * that has returned @a num_results results. Calls the `iter`
517 * from @a cls for each result.
519 * @param cls closure with the `struct ExtractResultContext`
520 * @param result the postgres result
521 * @param num_result the number of results in @a result
524 extract_result_cb (void *cls,
526 unsigned int num_results)
528 struct ExtractResultContext *erc = cls;
530 if (NULL == erc->iter)
532 for (unsigned int i = 0; i < num_results; i++)
534 struct GNUNET_TIME_Absolute expiration_time;
538 struct GNUNET_PeerIdentity *path;
540 struct GNUNET_HashCode key;
541 struct GNUNET_PQ_ResultSpec rs[] = {
542 GNUNET_PQ_result_spec_absolute_time ("",
544 GNUNET_PQ_result_spec_uint32 ("type",
546 GNUNET_PQ_result_spec_variable_size ("value",
549 GNUNET_PQ_result_spec_variable_size ("path",
552 GNUNET_PQ_result_spec_auto_from_type ("key",
554 GNUNET_PQ_result_spec_end
558 GNUNET_PQ_extract_result (result,
565 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
570 path_len %= sizeof(struct GNUNET_PeerIdentity);
571 LOG (GNUNET_ERROR_TYPE_DEBUG,
572 "Found result of size %u bytes and type %u in database\n",
573 (unsigned int) data_size,
574 (unsigned int) type);
576 erc->iter (erc->iter_cls,
580 (enum GNUNET_BLOCK_Type) type,
585 LOG (GNUNET_ERROR_TYPE_DEBUG,
586 "Ending iteration (client error)\n");
587 GNUNET_PQ_cleanup_result (rs);
590 GNUNET_PQ_cleanup_result (rs);
596 * Iterate over the results that are "close" to a particular key in
597 * the datacache. "close" is defined as numerically larger than @a
598 * key (when interpreted as a circular address space), with small
601 * @param cls closure (internal context for the plugin)
602 * @param key area of the keyspace to look into
603 * @param num_results number of results that should be returned to @a iter
604 * @param iter maybe NULL (to just count)
605 * @param iter_cls closure for @a iter
606 * @return the number of results found
609 postgres_plugin_get_closest (void *cls,
610 const struct GNUNET_HashCode *key,
611 unsigned int num_results,
612 GNUNET_DATACACHE_Iterator iter,
615 struct Plugin *plugin = cls;
616 uint32_t num_results32 = (uint32_t) num_results;
617 struct GNUNET_TIME_Absolute now;
618 struct GNUNET_PQ_QueryParam params[] = {
619 GNUNET_PQ_query_param_auto_from_type (key),
620 GNUNET_PQ_query_param_absolute_time (&now),
621 GNUNET_PQ_query_param_uint32 (&num_results32),
622 GNUNET_PQ_query_param_end
624 enum GNUNET_DB_QueryStatus res;
625 struct ExtractResultContext erc;
628 erc.iter_cls = iter_cls;
629 now = GNUNET_TIME_absolute_get ();
630 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
637 LOG (GNUNET_ERROR_TYPE_DEBUG,
638 "Ending iteration (postgres error)\n");
641 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
644 LOG (GNUNET_ERROR_TYPE_DEBUG,
645 "Ending iteration (no more results)\n");
653 * Entry point for the plugin.
655 * @param cls closure (the `struct GNUNET_DATACACHE_PluginEnvironmnet`)
656 * @return the plugin's closure (our `struct Plugin`)
659 libgnunet_plugin_datacache_postgres_init (void *cls)
661 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
662 struct GNUNET_DATACACHE_PluginFunctions *api;
663 struct Plugin *plugin;
665 plugin = GNUNET_new (struct Plugin);
668 if (GNUNET_OK != init_connection (plugin))
670 GNUNET_free (plugin);
674 api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
676 api->get = &postgres_plugin_get;
677 api->put = &postgres_plugin_put;
678 api->del = &postgres_plugin_del;
679 api->get_random = &postgres_plugin_get_random;
680 api->get_closest = &postgres_plugin_get_closest;
681 LOG (GNUNET_ERROR_TYPE_INFO,
682 "Postgres datacache running\n");
688 * Exit point from the plugin.
690 * @param cls closure (our `struct Plugin`)
694 libgnunet_plugin_datacache_postgres_done (void *cls)
696 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
697 struct Plugin *plugin = api->cls;
699 GNUNET_PQ_disconnect (plugin->dbh);
700 GNUNET_free (plugin);
706 /* end of plugin_datacache_postgres.c */