From afec51a8dd5d9f77a00d902acd7b5efc04675298 Mon Sep 17 00:00:00 2001 From: "Nathan S. Evans" Date: Wed, 21 Jul 2010 16:56:05 +0000 Subject: [PATCH] plugin to write sql queries to file instead of connecting to server --- src/dht/plugin_dhtlog_mysql_dump.c | 545 +++++++++++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 src/dht/plugin_dhtlog_mysql_dump.c diff --git a/src/dht/plugin_dhtlog_mysql_dump.c b/src/dht/plugin_dhtlog_mysql_dump.c new file mode 100644 index 000000000..c71f38c87 --- /dev/null +++ b/src/dht/plugin_dhtlog_mysql_dump.c @@ -0,0 +1,545 @@ +/* + This file is part of GNUnet. + (C) 2006 - 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file src/dht/plugin_dhtlog_mysql.c + * @brief MySQL logging plugin to record DHT operations to MySQL server, + * but write all queries to file instead of the actual server + * so that they can be imported later. The idea is that connecting + * to the MySQL server X times can be really problematic, but hopefully + * writing to a single file is more reliable. + * @author Nathan Evans + * + * Database: MySQL + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "dhtlog.h" + + +#define DEBUG_DHTLOG GNUNET_NO + +/** + * Maximum number of supported parameters for a prepared + * statement. Increase if needed. + */ +#define MAX_PARAM 32 + + +static unsigned long max_varchar_len; + +/** + * The configuration the DHT service is running with + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +#define INSERT_QUERIES_STMT "prepare insert_query from 'INSERT INTO queries (trialuid, querytype, hops, dhtkeyuid, dhtqueryid, succeeded, nodeuid) "\ + "VALUES (@temp_trial, ?, ?, ?, ?, ?, ?)'" + +#define INSERT_ROUTES_STMT "prepare insert_route from 'INSERT INTO routes (trialuid, querytype, hops, dhtkeyuid, dhtqueryid, succeeded, nodeuid, from_node, to_node) "\ + "VALUES (@temp_trial, ?, ?, ?, ?, ?, ?, ?, ?)'" + +#define INSERT_NODES_STMT "prepare insert_node from 'INSERT INTO nodes (trialuid, nodeid) "\ + "VALUES (@temp_trial, ?)'" + +#define INSERT_TRIALS_STMT "prepare insert_trial from 'INSERT INTO trials"\ + "(starttime, numnodes, topology,"\ + "topology_percentage, topology_probability,"\ + "blacklist_topology, connect_topology, connect_topology_option,"\ + "connect_topology_option_modifier, puts, gets, "\ + "concurrent, settle_time, num_rounds, malicious_getters,"\ + "malicious_putters, malicious_droppers, message) "\ + "VALUES (NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'" + +#define INSERT_DHTKEY_STMT "prepare insert_dhtkey from 'INSERT ignore INTO dhtkeys (dhtkey, trialuid) "\ + "VALUES (?, @temp_trial)'" + +#define UPDATE_TRIALS_STMT "prepare update_trial from 'UPDATE trials set endtime=NOW(), total_messages_dropped = ?, total_bytes_dropped = ?, unknownPeers = ? where trialuid = @temp_trial'" + +#define UPDATE_CONNECTIONS_STMT "prepare update_conn from 'UPDATE trials set totalConnections = ? where trialuid = @temp_trial'" + +#define GET_TRIAL_STMT "prepare select_trial from 'SELECT MAX( trialuid ) FROM trials into @temp_trial'" + +#define GET_DHTKEYUID_STMT "prepare get_dhtkeyuid from 'SELECT dhtkeyuid FROM dhtkeys where dhtkey = ? and trialuid = @temp_trial'" + +#define GET_NODEUID_STMT "prepare get_nodeuid from 'SELECT nodeuid FROM nodes where trialuid = @temp_trial and nodeid = ?'" + +/** + * File to dump all sql statements to. + */ +FILE *outfile; + +/** + * Create a prepared statement. + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure + */ +static int +prepared_statement_create (const char *statement) +{ + if (fprintf(outfile, "%s;\n", statement) > 0) + return GNUNET_OK; + + return GNUNET_SYSERR; +} + +/* + * Initialize the prepared statements for use with dht test logging + */ +static int +iopen () +{ +#define PINIT(a) (GNUNET_OK != (prepared_statement_create(a))) + if (PINIT (INSERT_QUERIES_STMT) || + PINIT (INSERT_ROUTES_STMT) || + PINIT (INSERT_TRIALS_STMT) || + PINIT (INSERT_NODES_STMT) || + PINIT (INSERT_DHTKEY_STMT) || + PINIT (UPDATE_TRIALS_STMT) || + PINIT (GET_DHTKEYUID_STMT) || + PINIT (GET_NODEUID_STMT) || + PINIT (UPDATE_CONNECTIONS_STMT) || + PINIT (GET_TRIAL_STMT)) + { + return GNUNET_SYSERR; + } +#undef PINIT + return GNUNET_OK; +} + + +/* + * Inserts the specified trial into the dhttests.trials table + * + * @param trialuid return the trialuid of the newly inserted trial + * @param num_nodes how many nodes are in the trial + * @param topology integer representing topology for this trial + * @param blacklist_topology integer representing blacklist topology for this trial + * @param connect_topology integer representing connect topology for this trial + * @param connect_topology_option integer representing connect topology option + * @param connect_topology_option_modifier float to modify connect option + * @param topology_percentage percentage modifier for certain topologies + * @param topology_probability probability modifier for certain topologies + * @param puts number of puts to perform + * @param gets number of gets to perform + * @param concurrent number of concurrent requests + * @param settle_time time to wait between creating topology and starting testing + * @param num_rounds number of times to repeat the trial + * @param malicious_getters number of malicious GET peers in the trial + * @param malicious_putters number of malicious PUT peers in the trial + * @param malicious_droppers number of malicious DROP peers in the trial + * @param message string to put into DB for this trial + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure + */ +int +add_trial (unsigned long long *trialuid, int num_nodes, int topology, + int blacklist_topology, int connect_topology, + int connect_topology_option, float connect_topology_option_modifier, + float topology_percentage, float topology_probability, + int puts, int gets, int concurrent, int settle_time, + int num_rounds, int malicious_getters, int malicious_putters, + int malicious_droppers, char *message) +{ + int ret; + *trialuid = 0; + if (outfile == NULL) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "set @num = %d, @topology = %d, @bl = %d, " + "@connect = %d, @c_t_o = %d, @c_t_o_m = %f, @t_p = %f, " + "@t_pr = %f, @puts = %d, @gets = %d, " + "@concurrent = %d, @settle = %d, @rounds = %d, " + "@m_gets = %d, @m_puts = %d, @m_drops = %d, " + "@message = \"%s\";\n", num_nodes, topology, + blacklist_topology, connect_topology, + connect_topology_option, connect_topology_option_modifier, + topology_percentage, topology_probability, + puts, gets, concurrent, settle_time, + num_rounds, malicious_getters, malicious_putters, + malicious_droppers, message); + + if (ret < 0) + return GNUNET_SYSERR; + ret = fprintf(outfile, "execute insert_trial using " + "@num, @topology, @bl, " + "@connect, @c_t_o, @c_t_o_m, @t_p, " + "@t_pr, @puts, @gets, " + "@concurrent, @settle, @rounds, " + "@m_gets, @m_puts, @m_drops, " + "@message;\n"); + + ret = fprintf(outfile, "execute select_trial;\n"); + + if (ret >= 0) + return GNUNET_OK; + return GNUNET_SYSERR; +} + + +/* + * Inserts the specified dhtkey into the dhttests.dhtkeys table, + * stores return value of dhttests.dhtkeys.dhtkeyuid into dhtkeyuid + * + * @param dhtkeyuid return value + * @param dhtkey hashcode of key to insert + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure + */ +int +add_dhtkey (unsigned long long *dhtkeyuid, const GNUNET_HashCode * dhtkey) +{ + int ret; + *dhtkeyuid = 0; + + if (outfile == NULL) + return GNUNET_SYSERR; + + if (dhtkey != NULL) + ret = fprintf(outfile, "set @dhtkey = \"%s\";\n", GNUNET_h2s_full(dhtkey)); + else + ret = fprintf(outfile, "set @dhtkey = XXXXX;\n"); + + if (ret < 0) + return GNUNET_SYSERR; + ret = fprintf(outfile, "execute insert_dhtkey using @dhtkey;\n"); + + if (ret >= 0) + return GNUNET_OK; + return GNUNET_SYSERR; +} + +/* + * Inserts the specified node into the dhttests.nodes table + * + * @param nodeuid the inserted node uid + * @param node the node to insert + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure + */ +int +add_node (unsigned long long *nodeuid, struct GNUNET_PeerIdentity * node) +{ + int ret; + + if (node == NULL) + return GNUNET_SYSERR; + + if (outfile == NULL) + return GNUNET_SYSERR; + + if (node != NULL) + ret = fprintf(outfile, "set @node = \"%s\";\n", GNUNET_h2s_full(&node->hashPubKey)); + else + return GNUNET_SYSERR; + + if (ret < 0) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "execute insert_node using @node;\n"); + + if (ret >= 0) + return GNUNET_OK; + return GNUNET_SYSERR; +} + +/* + * Update dhttests.trials table with current server time as end time + * + * @param trialuid trial to update + * @param totalMessagesDropped stats value for messages dropped + * @param totalBytesDropped stats value for total bytes dropped + * @param unknownPeers stats value for unknown peers + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure. + */ +int +update_trials (unsigned long long trialuid, + unsigned long long totalMessagesDropped, + unsigned long long totalBytesDropped, + unsigned long long unknownPeers) +{ + int ret; +#if DEBUG_DHTLOG + if (trialuid != current_trial) + { + fprintf (stderr, + _("Trialuid to update is not equal to current_trial\n")); + } +#endif + + if (outfile == NULL) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "set @m_dropped = %llu, @b_dropped = %llu, @unknown = %llu;\n", totalMessagesDropped, totalBytesDropped, unknownPeers); + + if (ret < 0) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "execute update_trial using @m_dropped, @b_dropped, @unknown;\n"); + + if (ret >= 0) + return GNUNET_OK; + else + return GNUNET_SYSERR; +} + + +/* + * Update dhttests.trials table with total connections information + * + * @param trialuid the trialuid to update + * @param totalConnections the number of connections + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure. + */ +int +add_connections (unsigned long long trialuid, unsigned int totalConnections) +{ + int ret; +#if DEBUG_DHTLOG + if (trialuid != current_trial) + { + fprintf (stderr, + _("Trialuid to update is not equal to current_trial(!)(?)\n")); + } +#endif + if (outfile == NULL) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "set @conns = %u;\n", totalConnections); + + if (ret < 0) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "execute update_connections using @conns;\n"); + + if (ret >= 0) + return GNUNET_OK; + else + return GNUNET_SYSERR; +} + +/* + * Inserts the specified query into the dhttests.queries table + * + * @param sqlqueruid inserted query uid + * @param queryid dht query id + * @param type type of the query + * @param hops number of hops query traveled + * @param succeeded whether or not query was successful + * @param node the node the query hit + * @param key the key of the query + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure. + */ +int +add_query (unsigned long long *sqlqueryuid, unsigned long long queryid, + unsigned int type, unsigned int hops, int succeeded, + const struct GNUNET_PeerIdentity * node, const GNUNET_HashCode * key) +{ + int ret; + + if (outfile == NULL) + return GNUNET_SYSERR; + + *sqlqueryuid = 0; + + if (key != NULL) + ret = fprintf(outfile, "select dhtkeyuid from dhtkeys where trialuid = @temp_trial and dhtkey = \"%s\" into @temp_dhtkey;\n", GNUNET_h2s_full(key)); + else + ret = fprintf(outfile, "set @temp_dhtkey = 0;\n"); + + if (node != NULL) + ret = fprintf(outfile, "select nodeuid from nodes where trialuid = @temp_trial and nodeid = \"%s\" into @temp_node;\n", GNUNET_h2s_full(&node->hashPubKey)); + else + ret = fprintf(outfile, "set @temp_node = 0;\n"); + + ret = fprintf(outfile, "set @qid = %llu, @type = %u, @hops = %u, @succ = %d;\n", queryid, type, hops, succeeded); + + if (ret < 0) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "execute insert_query using @type, @hops, @temp_dhtkey, @qid, @succ, @temp_node;\n"); + + if (ret >= 0) + return GNUNET_OK; + else + return GNUNET_SYSERR; +} + +/* + * Inserts the specified route information into the dhttests.routes table + * + * @param sqlqueruid inserted query uid + * @param queryid dht query id + * @param type type of the query + * @param hops number of hops query traveled + * @param succeeded whether or not query was successful + * @param node the node the query hit + * @param key the key of the query + * @param from_node the node that sent the message to node + * @param to_node next node to forward message to + * + * @return GNUNET_OK on success, GNUNET_SYSERR on failure. + */ +int +add_route (unsigned long long *sqlqueryuid, unsigned long long queryid, + unsigned int type, unsigned int hops, + int succeeded, const struct GNUNET_PeerIdentity * node, + const GNUNET_HashCode * key, const struct GNUNET_PeerIdentity * from_node, + const struct GNUNET_PeerIdentity * to_node) +{ + int ret; + + if (outfile == NULL) + return GNUNET_SYSERR; + + *sqlqueryuid = 0; + + if (key != NULL) + ret = fprintf(outfile, "select dhtkeyuid from dhtkeys where trialuid = @temp_trial and dhtkey = \"%s\" into @temp_dhtkey;\n", GNUNET_h2s_full(key)); + else + ret = fprintf(outfile, "set @temp_dhtkey = 0;\n"); + + if (node != NULL) + ret = fprintf(outfile, "select nodeuid from nodes where trialuid = @temp_trial and nodeid = \"%s\" into @temp_node;\n", GNUNET_h2s_full(&node->hashPubKey)); + else + ret = fprintf(outfile, "set @temp_node = 0;\n"); + + if (from_node != NULL) + ret = fprintf(outfile, "select nodeuid from nodes where trialuid = @temp_trial and nodeid = \"%s\" into @temp_from_node;\n", GNUNET_h2s_full(&from_node->hashPubKey)); + else + ret = fprintf(outfile, "set @temp_from_node = 0;\n"); + + if (to_node != NULL) + ret = fprintf(outfile, "select nodeuid from nodes where trialuid = @temp_trial and nodeid = \"%s\" into @temp_to_node;\n", GNUNET_h2s_full(&to_node->hashPubKey)); + else + ret = fprintf(outfile, "set @temp_to_node = 0;\n"); + + ret = fprintf(outfile, "set @qid = %llu, @type = %u, @hops = %u, @succ = %d;\n", queryid, type, hops, succeeded); + + if (ret < 0) + return GNUNET_SYSERR; + + ret = fprintf(outfile, "execute insert_route using @type, @hops, @temp_dhtkey, @qid, @succ, @temp_node, @temp_from_node, @temp_to_node;\n"); + + if (ret >= 0) + return GNUNET_OK; + else + return GNUNET_SYSERR; +} + +/* + * Provides the dhtlog api + * + * @param c the configuration to use to connect to a server + * + * @return the handle to the server, or NULL on error + */ +void * +libgnunet_plugin_dhtlog_mysql_dump_init (void * cls) +{ + struct GNUNET_DHTLOG_Plugin *plugin = cls; + char *outfile_name; + char *outfile_path; + char *fn; + int dirwarn; + + cfg = plugin->cfg; + max_varchar_len = 255; + +#if DEBUG_DHTLOG + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MySQL (DUMP) DHT Logger: initializing\n"); +#endif + + if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (plugin->cfg, + "MYSQLDUMP", "PATH", + &outfile_path)) + { + outfile_path = GNUNET_strdup(""); + } + + GNUNET_asprintf (&outfile_name, + "%s%s-%d", + outfile_path, + "mysqldump", + getpid()); + + fn = GNUNET_STRINGS_filename_expand (outfile_name); + + dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)); + outfile = FOPEN (fn, "w"); + + if (outfile == NULL) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn); + if (dirwarn) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to create or access directory for log file `%s'\n"), + fn); + GNUNET_free(outfile_path); + GNUNET_free(outfile_name); + GNUNET_free (fn); + return NULL; + } + + GNUNET_free (outfile_path); + GNUNET_free (outfile_name); + GNUNET_free (fn); + + if (iopen () != GNUNET_OK) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to create file for dhtlog.\n")); + return NULL; + } + GNUNET_assert(plugin->dhtlog_api == NULL); + plugin->dhtlog_api = GNUNET_malloc(sizeof(struct GNUNET_DHTLOG_Handle)); + plugin->dhtlog_api->insert_trial = &add_trial; + plugin->dhtlog_api->insert_query = &add_query; + plugin->dhtlog_api->update_trial = &update_trials; + plugin->dhtlog_api->insert_route = &add_route; + plugin->dhtlog_api->insert_node = &add_node; + plugin->dhtlog_api->insert_dhtkey = &add_dhtkey; + plugin->dhtlog_api->update_connections = &add_connections; + + return NULL; +} + +/** + * Shutdown the plugin. + */ +void * +libgnunet_plugin_dhtlog_mysql_dump_done (void * cls) +{ + struct GNUNET_DHTLOG_Handle *dhtlog_api = cls; +#if DEBUG_DHTLOG + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "MySQL DHT Logger: database shutdown\n"); +#endif + GNUNET_assert(dhtlog_api != NULL); + + GNUNET_free(dhtlog_api); + return NULL; +} + +/* end of plugin_dhtlog_mysql.c */ -- 2.25.1