2 This file is part of GNUnet
3 Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @file datacache/plugin_datacache_postgres.c
23 * @brief postgres for an implementation of a database backend for the datacache
24 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_pq_lib.h"
29 #include "gnunet_datacache_plugin.h"
31 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache-postgres", __VA_ARGS__)
34 * Per-entry overhead estimate
36 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 24)
39 * Context for all functions in this plugin.
44 * Our execution environment.
46 struct GNUNET_DATACACHE_PluginEnvironment *env;
49 * Native Postgres database handle.
54 * Number of key-value pairs in the database.
56 unsigned int num_items;
61 * @brief Get a database handle
63 * @param plugin global context
64 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
67 init_connection (struct Plugin *plugin)
69 struct GNUNET_PQ_ExecuteStatement es[] = {
70 GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn011dc ("
71 " type INTEGER NOT NULL,"
72 " prox INTEGER NOT NULL,"
73 " discard_time BIGINT NOT NULL,"
74 " key BYTEA NOT NULL,"
75 " value BYTEA NOT NULL,"
76 " path BYTEA DEFAULT NULL)"
78 GNUNET_PQ_make_try_execute (
79 "CREATE INDEX IF NOT EXISTS idx_key ON gn011dc (key)"),
80 GNUNET_PQ_make_try_execute (
81 "CREATE INDEX IF NOT EXISTS idx_dt ON gn011dc (discard_time)"),
82 GNUNET_PQ_make_execute (
83 "ALTER TABLE gn011dc ALTER value SET STORAGE EXTERNAL"),
84 GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER key SET STORAGE PLAIN"),
85 GNUNET_PQ_EXECUTE_STATEMENT_END
87 struct GNUNET_PQ_PreparedStatement ps[] = {
88 GNUNET_PQ_make_prepare ("getkt",
89 "SELECT discard_time,type,value,path FROM gn011dc "
90 "WHERE key=$1 AND type=$2 AND discard_time >= $3",
92 GNUNET_PQ_make_prepare ("getk",
93 "SELECT discard_time,type,value,path FROM gn011dc "
94 "WHERE key=$1 AND discard_time >= $2",
96 GNUNET_PQ_make_prepare ("getex",
97 "SELECT length(value) AS len,oid,key FROM gn011dc"
98 " WHERE discard_time < $1"
99 " ORDER BY discard_time ASC LIMIT 1",
101 GNUNET_PQ_make_prepare ("getm",
102 "SELECT length(value) AS len,oid,key FROM gn011dc"
103 " ORDER BY prox ASC, discard_time ASC LIMIT 1",
105 GNUNET_PQ_make_prepare ("get_random",
106 "SELECT discard_time,type,value,path,key FROM gn011dc"
107 " WHERE discard_time >= $1"
108 " ORDER BY key ASC LIMIT 1 OFFSET $2",
110 GNUNET_PQ_make_prepare ("get_closest",
111 "SELECT discard_time,type,value,path,key FROM gn011dc "
112 "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3",
114 GNUNET_PQ_make_prepare ("delrow",
115 "DELETE FROM gn011dc WHERE oid=$1",
117 GNUNET_PQ_make_prepare ("put",
118 "INSERT INTO gn011dc (type, prox, discard_time, key, value, path) "
119 "VALUES ($1, $2, $3, $4, $5, $6)",
121 GNUNET_PQ_PREPARED_STATEMENT_END
124 plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
125 "datacache-postgres");
126 if (NULL == plugin->dbh)
127 return GNUNET_SYSERR;
129 GNUNET_PQ_exec_statements (plugin->dbh,
132 PQfinish (plugin->dbh);
134 return GNUNET_SYSERR;
138 GNUNET_PQ_prepare_statements (plugin->dbh,
141 PQfinish (plugin->dbh);
143 return GNUNET_SYSERR;
150 * Store an item in the datastore.
152 * @param cls closure (our `struct Plugin`)
153 * @param key key to store @a data under
154 * @param prox proximity of @a key to my PID
155 * @param data_size number of bytes in @a data
156 * @param data data to store
157 * @param type type of the value
158 * @param discard_time when to discard the value in any case
159 * @param path_info_len number of entries in @a path_info
160 * @param path_info a path through the network
161 * @return 0 if duplicate, -1 on error, number of bytes used otherwise
164 postgres_plugin_put (void *cls,
165 const struct GNUNET_HashCode *key,
169 enum GNUNET_BLOCK_Type type,
170 struct GNUNET_TIME_Absolute discard_time,
171 unsigned int path_info_len,
172 const struct GNUNET_PeerIdentity *path_info)
174 struct Plugin *plugin = cls;
175 uint32_t type32 = (uint32_t) type;
176 struct GNUNET_PQ_QueryParam params[] = {
177 GNUNET_PQ_query_param_uint32 (&type32),
178 GNUNET_PQ_query_param_uint32 (&prox),
179 GNUNET_PQ_query_param_absolute_time (&discard_time),
180 GNUNET_PQ_query_param_auto_from_type (key),
181 GNUNET_PQ_query_param_fixed_size (data, data_size),
182 GNUNET_PQ_query_param_fixed_size (path_info,
183 path_info_len * sizeof(struct
184 GNUNET_PeerIdentity)),
185 GNUNET_PQ_query_param_end
187 enum GNUNET_DB_QueryStatus ret;
189 ret = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
195 return data_size + OVERHEAD;
200 * Closure for #handle_results.
202 struct HandleResultContext
205 * Function to call on each result, may be NULL.
207 GNUNET_DATACACHE_Iterator iter;
210 * Closure for @e iter.
217 const struct GNUNET_HashCode *key;
222 * Function to be called with the results of a SELECT statement
223 * that has returned @a num_results results. Parse the result
224 * and call the callback given in @a cls
226 * @param cls closure of type `struct HandleResultContext`
227 * @param result the postgres result
228 * @param num_result the number of results in @a result
231 handle_results (void *cls,
233 unsigned int num_results)
235 struct HandleResultContext *hrc = cls;
237 for (unsigned int i = 0; i < num_results; i++)
239 struct GNUNET_TIME_Absolute expiration_time;
243 struct GNUNET_PeerIdentity *path;
245 struct GNUNET_PQ_ResultSpec rs[] = {
246 GNUNET_PQ_result_spec_absolute_time ("discard_time",
248 GNUNET_PQ_result_spec_uint32 ("type",
250 GNUNET_PQ_result_spec_variable_size ("value",
253 GNUNET_PQ_result_spec_variable_size ("path",
256 GNUNET_PQ_result_spec_end
260 GNUNET_PQ_extract_result (result,
267 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
272 path_len %= sizeof(struct GNUNET_PeerIdentity);
273 LOG (GNUNET_ERROR_TYPE_DEBUG,
274 "Found result of size %u bytes and type %u in database\n",
275 (unsigned int) data_size,
276 (unsigned int) type);
277 if ((NULL != hrc->iter) &&
279 hrc->iter (hrc->iter_cls,
283 (enum GNUNET_BLOCK_Type) type,
288 LOG (GNUNET_ERROR_TYPE_DEBUG,
289 "Ending iteration (client error)\n");
290 GNUNET_PQ_cleanup_result (rs);
293 GNUNET_PQ_cleanup_result (rs);
299 * Iterate over the results for a particular key
302 * @param cls closure (our `struct Plugin`)
303 * @param key key to look for
304 * @param type entries of which type are relevant?
305 * @param iter maybe NULL (to just count)
306 * @param iter_cls closure for @a iter
307 * @return the number of results found
310 postgres_plugin_get (void *cls,
311 const struct GNUNET_HashCode *key,
312 enum GNUNET_BLOCK_Type type,
313 GNUNET_DATACACHE_Iterator iter,
316 struct Plugin *plugin = cls;
317 uint32_t type32 = (uint32_t) type;
318 struct GNUNET_TIME_Absolute now;
319 struct GNUNET_PQ_QueryParam paramk[] = {
320 GNUNET_PQ_query_param_auto_from_type (key),
321 GNUNET_PQ_query_param_absolute_time (&now),
322 GNUNET_PQ_query_param_end
324 struct GNUNET_PQ_QueryParam paramkt[] = {
325 GNUNET_PQ_query_param_auto_from_type (key),
326 GNUNET_PQ_query_param_uint32 (&type32),
327 GNUNET_PQ_query_param_absolute_time (&now),
328 GNUNET_PQ_query_param_end
330 enum GNUNET_DB_QueryStatus res;
331 struct HandleResultContext hr_ctx;
333 now = GNUNET_TIME_absolute_get ();
335 hr_ctx.iter_cls = iter_cls;
337 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
338 (0 == type) ? "getk" : "getkt",
339 (0 == type) ? paramk : paramkt,
349 * Delete the entry with the lowest expiration value
350 * from the datacache right now.
352 * @param cls closure (our `struct Plugin`)
353 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
356 postgres_plugin_del (void *cls)
358 struct Plugin *plugin = cls;
359 struct GNUNET_PQ_QueryParam pempty[] = {
360 GNUNET_PQ_query_param_end
364 struct GNUNET_HashCode key;
365 struct GNUNET_PQ_ResultSpec rs[] = {
366 GNUNET_PQ_result_spec_uint32 ("len",
368 GNUNET_PQ_result_spec_uint32 ("oid",
370 GNUNET_PQ_result_spec_auto_from_type ("key",
372 GNUNET_PQ_result_spec_end
374 enum GNUNET_DB_QueryStatus res;
375 struct GNUNET_PQ_QueryParam dparam[] = {
376 GNUNET_PQ_query_param_uint32 (&oid),
377 GNUNET_PQ_query_param_end
379 struct GNUNET_TIME_Absolute now;
380 struct GNUNET_PQ_QueryParam xparam[] = {
381 GNUNET_PQ_query_param_absolute_time (&now),
382 GNUNET_PQ_query_param_end
385 now = GNUNET_TIME_absolute_get ();
386 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
391 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
396 return GNUNET_SYSERR;
397 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
400 LOG (GNUNET_ERROR_TYPE_DEBUG,
401 "Ending iteration (no more results)\n");
404 res = GNUNET_PQ_eval_prepared_non_select (plugin->dbh,
409 GNUNET_PQ_cleanup_result (rs);
410 return GNUNET_SYSERR;
413 plugin->env->delete_notify (plugin->env->cls,
416 GNUNET_PQ_cleanup_result (rs);
422 * Obtain a random key-value pair from the datacache.
424 * @param cls closure (our `struct Plugin`)
425 * @param iter maybe NULL (to just count)
426 * @param iter_cls closure for @a iter
427 * @return the number of results found, zero (datacache empty) or one
430 postgres_plugin_get_random (void *cls,
431 GNUNET_DATACACHE_Iterator iter,
434 struct Plugin *plugin = cls;
436 struct GNUNET_TIME_Absolute now;
437 struct GNUNET_TIME_Absolute expiration_time;
441 struct GNUNET_PeerIdentity *path;
442 struct GNUNET_HashCode key;
444 enum GNUNET_DB_QueryStatus res;
445 struct GNUNET_PQ_QueryParam params[] = {
446 GNUNET_PQ_query_param_absolute_time (&now),
447 GNUNET_PQ_query_param_uint32 (&off),
448 GNUNET_PQ_query_param_end
450 struct GNUNET_PQ_ResultSpec rs[] = {
451 GNUNET_PQ_result_spec_absolute_time ("discard_time",
453 GNUNET_PQ_result_spec_uint32 ("type",
455 GNUNET_PQ_result_spec_variable_size ("value",
458 GNUNET_PQ_result_spec_variable_size ("path",
461 GNUNET_PQ_result_spec_auto_from_type ("key",
463 GNUNET_PQ_result_spec_end
466 if (0 == plugin->num_items)
470 now = GNUNET_TIME_absolute_get ();
471 off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
473 res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh,
482 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
487 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
492 path_len %= sizeof(struct GNUNET_PeerIdentity);
493 LOG (GNUNET_ERROR_TYPE_DEBUG,
494 "Found random value with key %s of size %u bytes and type %u in database\n",
496 (unsigned int) data_size,
497 (unsigned int) type);
498 (void) iter (iter_cls,
502 (enum GNUNET_BLOCK_Type) type,
506 GNUNET_PQ_cleanup_result (rs);
512 * Closure for #extract_result_cb.
514 struct ExtractResultContext
517 * Function to call for each result found.
519 GNUNET_DATACACHE_Iterator iter;
522 * Closure for @e iter.
529 * Function to be called with the results of a SELECT statement
530 * that has returned @a num_results results. Calls the `iter`
531 * from @a cls for each result.
533 * @param cls closure with the `struct ExtractResultContext`
534 * @param result the postgres result
535 * @param num_result the number of results in @a result
538 extract_result_cb (void *cls,
540 unsigned int num_results)
542 struct ExtractResultContext *erc = cls;
544 if (NULL == erc->iter)
546 for (unsigned int i = 0; i < num_results; i++)
548 struct GNUNET_TIME_Absolute expiration_time;
552 struct GNUNET_PeerIdentity *path;
554 struct GNUNET_HashCode key;
555 struct GNUNET_PQ_ResultSpec rs[] = {
556 GNUNET_PQ_result_spec_absolute_time ("",
558 GNUNET_PQ_result_spec_uint32 ("type",
560 GNUNET_PQ_result_spec_variable_size ("value",
563 GNUNET_PQ_result_spec_variable_size ("path",
566 GNUNET_PQ_result_spec_auto_from_type ("key",
568 GNUNET_PQ_result_spec_end
572 GNUNET_PQ_extract_result (result,
579 if (0 != (path_len % sizeof(struct GNUNET_PeerIdentity)))
584 path_len %= sizeof(struct GNUNET_PeerIdentity);
585 LOG (GNUNET_ERROR_TYPE_DEBUG,
586 "Found result of size %u bytes and type %u in database\n",
587 (unsigned int) data_size,
588 (unsigned int) type);
590 erc->iter (erc->iter_cls,
594 (enum GNUNET_BLOCK_Type) type,
599 LOG (GNUNET_ERROR_TYPE_DEBUG,
600 "Ending iteration (client error)\n");
601 GNUNET_PQ_cleanup_result (rs);
604 GNUNET_PQ_cleanup_result (rs);
610 * Iterate over the results that are "close" to a particular key in
611 * the datacache. "close" is defined as numerically larger than @a
612 * key (when interpreted as a circular address space), with small
615 * @param cls closure (internal context for the plugin)
616 * @param key area of the keyspace to look into
617 * @param num_results number of results that should be returned to @a iter
618 * @param iter maybe NULL (to just count)
619 * @param iter_cls closure for @a iter
620 * @return the number of results found
623 postgres_plugin_get_closest (void *cls,
624 const struct GNUNET_HashCode *key,
625 unsigned int num_results,
626 GNUNET_DATACACHE_Iterator iter,
629 struct Plugin *plugin = cls;
630 uint32_t num_results32 = (uint32_t) num_results;
631 struct GNUNET_TIME_Absolute now;
632 struct GNUNET_PQ_QueryParam params[] = {
633 GNUNET_PQ_query_param_auto_from_type (key),
634 GNUNET_PQ_query_param_absolute_time (&now),
635 GNUNET_PQ_query_param_uint32 (&num_results32),
636 GNUNET_PQ_query_param_end
638 enum GNUNET_DB_QueryStatus res;
639 struct ExtractResultContext erc;
642 erc.iter_cls = iter_cls;
643 now = GNUNET_TIME_absolute_get ();
644 res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh,
651 LOG (GNUNET_ERROR_TYPE_DEBUG,
652 "Ending iteration (postgres error)\n");
655 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
658 LOG (GNUNET_ERROR_TYPE_DEBUG,
659 "Ending iteration (no more results)\n");
667 * Entry point for the plugin.
669 * @param cls closure (the `struct GNUNET_DATACACHE_PluginEnvironmnet`)
670 * @return the plugin's closure (our `struct Plugin`)
673 libgnunet_plugin_datacache_postgres_init (void *cls)
675 struct GNUNET_DATACACHE_PluginEnvironment *env = cls;
676 struct GNUNET_DATACACHE_PluginFunctions *api;
677 struct Plugin *plugin;
679 plugin = GNUNET_new (struct Plugin);
682 if (GNUNET_OK != init_connection (plugin))
684 GNUNET_free (plugin);
688 api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
690 api->get = &postgres_plugin_get;
691 api->put = &postgres_plugin_put;
692 api->del = &postgres_plugin_del;
693 api->get_random = &postgres_plugin_get_random;
694 api->get_closest = &postgres_plugin_get_closest;
695 LOG (GNUNET_ERROR_TYPE_INFO,
696 "Postgres datacache running\n");
702 * Exit point from the plugin.
704 * @param cls closure (our `struct Plugin`)
708 libgnunet_plugin_datacache_postgres_done (void *cls)
710 struct GNUNET_DATACACHE_PluginFunctions *api = cls;
711 struct Plugin *plugin = api->cls;
713 PQfinish (plugin->dbh);
714 GNUNET_free (plugin);
721 /* end of plugin_datacache_postgres.c */