2 This file is part of GNUnet.
3 (C) 2006 - 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file src/dht/plugin_dhtlog_mysql_dump_load.c
23 * @brief MySQL logging plugin to record DHT operations to MySQL server,
24 * but write all queries to file instead of the actual server
25 * so that they can be imported later. Since the first attempt
26 * (writing out SQL queries) seemed rather time consuming on insert,
27 * this plugin writes out tab separated values instead.
29 * @author Nathan Evans
35 #include "gnunet_util_lib.h"
39 #define DEBUG_DHTLOG GNUNET_YES
42 * Maximum number of supported parameters for a prepared
43 * statement. Increase if needed.
48 static unsigned long max_varchar_len;
51 * The configuration the DHT service is running with
53 static const struct GNUNET_CONFIGURATION_Handle *cfg;
55 #define DATE_STR_SIZE 50
57 static unsigned int topology_count;
60 * File(s) to dump all sql statements to.
63 FILE *generic_stat_outfile;
69 FILE *extended_topology_outfile;
74 static char date[DATE_STR_SIZE];
79 memset (date, 0, DATE_STR_SIZE);
80 tmptr = localtime (&timetmp);
82 strftime (date, DATE_STR_SIZE, "%Y-%m-%d %H:%M:%S", tmptr);
90 * Records the current topology (number of connections, time, trial)
92 * @param num_connections how many connections are in the topology
94 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
97 add_topology (int num_connections)
101 return GNUNET_SYSERR;
103 ret = fprintf(outfile, "insert into topology (trialuid, date, connections) values (@temp_trial, \"%s\", %d);\n", get_sql_time(), num_connections);
105 return GNUNET_SYSERR;
106 ret = fprintf(outfile, "select max(topology_uid) from topology into @temp_topology;\n");
109 return GNUNET_SYSERR;
113 * Inserts the specified round into the dhttests.rounds table
115 * @param round_type the type of round that is being started
116 * @param round_count counter for the round (if applicable)
118 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
120 int add_round (unsigned int round_type, unsigned int round_count)
124 return GNUNET_SYSERR;
126 ret = fprintf(outfile, "insert into rounds (trialuid, round_type, round_count, starttime) values (@temp_trial, \"%u\", \"%u\", \"%s\");\n", round_type, round_count, get_sql_time());
130 return GNUNET_SYSERR;
135 * Records a connection between two peers in the current topology
137 * @param first one side of the connection
138 * @param second other side of the connection
140 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
143 add_extended_topology (const struct GNUNET_PeerIdentity *first, const struct GNUNET_PeerIdentity *second)
147 return GNUNET_SYSERR;
149 ret = fprintf(extended_topology_outfile, "insert into extended_topology (topology_uid, uid_first, uid_second) values (%u, %s,", topology_count, GNUNET_h2s_full(&first->hashPubKey));
151 return GNUNET_SYSERR;
152 ret = fprintf(extended_topology_outfile, "%s);\n", GNUNET_h2s_full(&second->hashPubKey));
156 return GNUNET_SYSERR;
161 * Inserts the specified trial into the dhttests.trials table
163 * @param trial_info struct containing the data to insert about this trial
165 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
167 int add_trial (struct GNUNET_DHTLOG_TrialInfo *trial_info)
171 return GNUNET_SYSERR;
173 ret = fprintf(outfile, "INSERT INTO trials "
174 "(starttime, other_trial_identifier, numnodes, topology,"
175 "blacklist_topology, connect_topology, connect_topology_option,"
176 "connect_topology_option_modifier, topology_percentage, topology_probability,"
178 "concurrent, settle_time, num_rounds, malicious_getters,"
179 "malicious_putters, malicious_droppers, malicious_get_frequency,"
180 "malicious_put_frequency, stop_closest, stop_found, strict_kademlia, "
181 "gets_succeeded, message) "
182 "VALUES (\"%s\", %u, %u, %u, %u, %u, %u, %f, %f, %f, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, \"%s\");\n",
183 get_sql_time(), trial_info->other_identifier, trial_info->num_nodes, trial_info->topology,
184 trial_info->blacklist_topology, trial_info->connect_topology,
185 trial_info->connect_topology_option, trial_info->connect_topology_option_modifier,
186 trial_info->topology_percentage, trial_info->topology_probability,
187 trial_info->puts, trial_info->gets, trial_info->concurrent, trial_info->settle_time,
188 trial_info->num_rounds, trial_info->malicious_getters, trial_info->malicious_putters,
189 trial_info->malicious_droppers, trial_info->malicious_get_frequency, trial_info->malicious_put_frequency,
190 trial_info->stop_closest, trial_info->stop_found, trial_info->strict_kademlia, trial_info->gets_succeeded, trial_info->message);
193 return GNUNET_SYSERR;
195 ret = fprintf(outfile, "SELECT MAX( trialuid ) FROM trials into @temp_trial;\n");
200 return GNUNET_SYSERR;
205 * Inserts the specified stats into the dhttests.generic_stats table
207 * @param peer the peer inserting the statistic
208 * @param name the name of the statistic
209 * @param section the section of the statistic
210 * @param value the value of the statistic
212 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
215 add_generic_stat (const struct GNUNET_PeerIdentity *peer,
217 const char *section, uint64_t value)
220 return GNUNET_SYSERR;
223 fprintf(generic_stat_outfile, "TRIALUID\t%s\t%s\t%s\t%llu\n", GNUNET_h2s_full(&peer->hashPubKey), section, name, (unsigned long long)value);
230 * Inserts the specified stats into the dhttests.node_statistics table
232 * @param peer the peer inserting the statistic
233 * @param route_requests route requests seen
234 * @param route_forwards route requests forwarded
235 * @param result_requests route result requests seen
236 * @param client_requests client requests initiated
237 * @param result_forwards route results forwarded
238 * @param gets get requests handled
239 * @param puts put requests handle
240 * @param data_inserts data inserted at this node
241 * @param find_peer_requests find peer requests seen
242 * @param find_peers_started find peer requests initiated at this node
243 * @param gets_started get requests initiated at this node
244 * @param puts_started put requests initiated at this node
245 * @param find_peer_responses_received find peer responses received locally
246 * @param get_responses_received get responses received locally
247 * @param find_peer_responses_sent find peer responses sent from this node
248 * @param get_responses_sent get responses sent from this node
250 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
253 add_stat (const struct GNUNET_PeerIdentity *peer, unsigned int route_requests,
254 unsigned int route_forwards, unsigned int result_requests,
255 unsigned int client_requests, unsigned int result_forwards,
256 unsigned int gets, unsigned int puts,
257 unsigned int data_inserts, unsigned int find_peer_requests,
258 unsigned int find_peers_started, unsigned int gets_started,
259 unsigned int puts_started, unsigned int find_peer_responses_received,
260 unsigned int get_responses_received, unsigned int find_peer_responses_sent,
261 unsigned int get_responses_sent)
265 return GNUNET_SYSERR;
268 ret = fprintf(stat_outfile, "TRIALUID\t%s\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\n",
269 GNUNET_h2s_full(&peer->hashPubKey),
270 route_requests, route_forwards, result_requests,
271 client_requests, result_forwards, gets, puts,
272 data_inserts, find_peer_requests, find_peers_started,
273 gets_started, puts_started, find_peer_responses_received,
274 get_responses_received, find_peer_responses_sent,
278 return GNUNET_SYSERR;
283 return GNUNET_SYSERR;
286 * Inserts the specified dhtkey into the dhttests.dhtkeys table,
287 * stores return value of dhttests.dhtkeys.dhtkeyuid into dhtkeyuid
289 * @param dhtkeyuid return value
290 * @param dhtkey hashcode of key to insert
292 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
295 add_dhtkey (unsigned long long *dhtkeyuid, const GNUNET_HashCode * dhtkey)
298 if (dhtkeyuid != NULL)
301 if ((dhtkey_outfile == NULL) || (dhtkey == NULL))
302 return GNUNET_SYSERR;
304 ret = fprintf(dhtkey_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(dhtkey));
309 return GNUNET_SYSERR;
313 * Inserts the specified node into the dhttests.nodes table
315 * @param nodeuid the inserted node uid
316 * @param node the node to insert
318 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
321 add_node (unsigned long long *nodeuid, struct GNUNET_PeerIdentity * node)
325 if ((node == NULL) || (node_outfile == NULL))
326 return GNUNET_SYSERR;
328 ret = fprintf(node_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(&node->hashPubKey));
332 return GNUNET_SYSERR;
336 * Update dhttests.trials table with current server time as end time
338 * @param gets_succeeded how many gets did the testcase report as successful
340 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
343 update_trials (unsigned int gets_succeeded)
348 return GNUNET_SYSERR;
350 ret = fprintf(outfile, "update trials set endtime=\"%s\", gets_succeeded=%u where trialuid = @temp_trial;\n", get_sql_time(), gets_succeeded);
355 return GNUNET_SYSERR;
360 * Update dhttests.nodes table setting the identified
361 * node as a malicious dropper.
363 * @param peer the peer that was set to be malicious
365 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
368 set_malicious (struct GNUNET_PeerIdentity *peer)
373 return GNUNET_SYSERR;
375 ret = fprintf(outfile, "update nodes set malicious_dropper = 1 where trialuid = @temp_trial and nodeid = \"%s\";\n", GNUNET_h2s_full(&peer->hashPubKey));
380 return GNUNET_SYSERR;
385 * Update dhttests.trials table with total connections information
387 * @param totalConnections the number of connections
389 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
392 add_connections (unsigned int totalConnections)
397 return GNUNET_SYSERR;
399 ret = fprintf(outfile, "update trials set totalConnections = %u where trialuid = @temp_trial;\n", totalConnections);
404 return GNUNET_SYSERR;
409 * Update dhttests.topology table with total connections information
411 * @param connections the number of connections
413 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
416 update_topology (unsigned int connections)
420 return GNUNET_SYSERR;
422 ret = fprintf(outfile, "update topology set connections = %u where topology_uid = @temp_topology;\n", connections);
427 return GNUNET_SYSERR;
431 * Inserts the specified query into the dhttests.queries table
433 * @param sqlqueruid inserted query uid
434 * @param queryid dht query id
435 * @param type type of the query
436 * @param hops number of hops query traveled
437 * @param succeeded whether or not query was successful
438 * @param node the node the query hit
439 * @param key the key of the query
441 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
444 add_query (unsigned long long *sqlqueryuid, unsigned long long queryid,
445 unsigned int type, unsigned int hops, int succeeded,
446 const struct GNUNET_PeerIdentity * node, const GNUNET_HashCode * key)
450 if ((outfile == NULL) || (node == NULL) || (key == NULL))
451 return GNUNET_SYSERR;
453 if (sqlqueryuid != NULL)
456 ret = fprintf(query_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
459 return GNUNET_SYSERR;
461 ret = fprintf(query_outfile, "%s\t%llu\t%u\t%u\t%u\t%s\n", GNUNET_h2s_full(&node->hashPubKey), queryid, type, hops, succeeded, get_sql_time());
466 return GNUNET_SYSERR;
470 * Inserts the specified route information into the dhttests.routes table
472 * @param sqlqueruid inserted query uid
473 * @param queryid dht query id
474 * @param type type of the query
475 * @param hops number of hops query traveled
476 * @param succeeded whether or not query was successful
477 * @param node the node the query hit
478 * @param key the key of the query
479 * @param from_node the node that sent the message to node
480 * @param to_node next node to forward message to
482 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
485 add_route (unsigned long long *sqlqueryuid, unsigned long long queryid,
486 unsigned int type, unsigned int hops,
487 int succeeded, const struct GNUNET_PeerIdentity * node,
488 const GNUNET_HashCode * key, const struct GNUNET_PeerIdentity * from_node,
489 const struct GNUNET_PeerIdentity * to_node)
493 if ((outfile == NULL) || (node == NULL) || (key == NULL))
494 return GNUNET_SYSERR;
496 if (sqlqueryuid != NULL)
499 ret = fprintf(route_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
501 return GNUNET_SYSERR;
503 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&node->hashPubKey));
505 return GNUNET_SYSERR;
506 if (from_node == NULL)
507 ret = fprintf(route_outfile, "0\t");
509 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&from_node->hashPubKey));
511 return GNUNET_SYSERR;
514 ret = fprintf(route_outfile, "0\t%llu\t%u\t%u\t%d\n", queryid, type, hops, succeeded);
516 ret = fprintf(route_outfile, "%s\t%llu\t%u\t%u\t%d\n", GNUNET_h2s_full(&to_node->hashPubKey), queryid, type, hops, succeeded);
521 return GNUNET_SYSERR;
525 * Provides the dhtlog api
527 * @param c the configuration to use to connect to a server
529 * @return the handle to the server, or NULL on error
532 libgnunet_plugin_dhtlog_mysql_dump_load_init (void * cls)
534 struct GNUNET_DHTLOG_Plugin *plugin = cls;
541 max_varchar_len = 255;
543 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MySQL (DUMP) DHT Logger: initializing\n");
545 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (plugin->cfg,
549 outfile_path = GNUNET_strdup("");
552 GNUNET_asprintf (&outfile_name,
558 fn = GNUNET_STRINGS_filename_expand (outfile_name);
562 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
563 GNUNET_free(outfile_path);
564 GNUNET_free(outfile_name);
568 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
569 outfile = FOPEN (fn, "w");
573 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
575 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
576 _("Failed to create or access directory for log file `%s'\n"),
578 GNUNET_free(outfile_path);
579 GNUNET_free(outfile_name);
584 GNUNET_asprintf (&outfile_name,
590 fn = GNUNET_STRINGS_filename_expand (outfile_name);
594 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
595 GNUNET_free(outfile_path);
596 GNUNET_free(outfile_name);
600 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
601 node_outfile = FOPEN (fn, "w");
603 if (node_outfile == NULL)
605 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
607 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
608 _("Failed to create or access directory for log file `%s'\n"),
610 GNUNET_free(outfile_path);
611 GNUNET_free(outfile_name);
616 GNUNET_asprintf (&outfile_name,
623 fn = GNUNET_STRINGS_filename_expand (outfile_name);
627 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
628 GNUNET_free(outfile_path);
629 GNUNET_free(outfile_name);
633 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
634 route_outfile = FOPEN (fn, "w");
636 if (route_outfile == NULL)
638 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
640 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
641 _("Failed to create or access directory for log file `%s'\n"),
643 GNUNET_free(outfile_path);
644 GNUNET_free(outfile_name);
649 GNUNET_asprintf (&outfile_name,
656 fn = GNUNET_STRINGS_filename_expand (outfile_name);
660 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
661 GNUNET_free(outfile_path);
662 GNUNET_free(outfile_name);
666 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
667 query_outfile = FOPEN (fn, "w");
669 if (query_outfile == NULL)
671 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
673 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
674 _("Failed to create or access directory for log file `%s'\n"),
676 GNUNET_free(outfile_path);
677 GNUNET_free(outfile_name);
682 GNUNET_asprintf (&outfile_name,
689 fn = GNUNET_STRINGS_filename_expand (outfile_name);
693 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
694 GNUNET_free(outfile_path);
695 GNUNET_free(outfile_name);
699 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
700 stat_outfile = FOPEN (fn, "w");
702 if (stat_outfile == NULL)
704 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
706 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
707 _("Failed to create or access directory for log file `%s'\n"),
709 GNUNET_free(outfile_path);
710 GNUNET_free(outfile_name);
715 GNUNET_asprintf (&outfile_name,
718 "mysqldump_generic_stats",
721 fn = GNUNET_STRINGS_filename_expand (outfile_name);
725 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
726 GNUNET_free(outfile_path);
727 GNUNET_free(outfile_name);
731 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
732 generic_stat_outfile = FOPEN (fn, "w");
734 if (generic_stat_outfile == NULL)
736 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
738 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
739 _("Failed to create or access directory for log file `%s'\n"),
741 GNUNET_free(outfile_path);
742 GNUNET_free(outfile_name);
747 GNUNET_asprintf (&outfile_name,
753 fn = GNUNET_STRINGS_filename_expand (outfile_name);
757 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
758 GNUNET_free(outfile_path);
759 GNUNET_free(outfile_name);
763 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
764 dhtkey_outfile = FOPEN (fn, "w");
766 if (dhtkey_outfile == NULL)
768 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
770 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
771 _("Failed to create or access directory for log file `%s'\n"),
773 GNUNET_free(outfile_path);
774 GNUNET_free(outfile_name);
779 GNUNET_asprintf (&outfile_name,
782 "mysqldump_extended_topology",
785 fn = GNUNET_STRINGS_filename_expand (outfile_name);
789 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
790 GNUNET_free(outfile_path);
791 GNUNET_free(outfile_name);
795 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
796 extended_topology_outfile = FOPEN (fn, "w");
798 if (extended_topology_outfile == NULL)
800 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
802 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
803 _("Failed to create or access directory for log file `%s'\n"),
805 GNUNET_free(outfile_path);
806 GNUNET_free(outfile_name);
811 GNUNET_free (outfile_path);
812 GNUNET_free (outfile_name);
815 GNUNET_assert(plugin->dhtlog_api == NULL);
816 plugin->dhtlog_api = GNUNET_malloc(sizeof(struct GNUNET_DHTLOG_Handle));
817 plugin->dhtlog_api->insert_trial = &add_trial;
818 plugin->dhtlog_api->insert_stat = &add_stat;
819 plugin->dhtlog_api->insert_round = &add_round;
820 plugin->dhtlog_api->insert_query = &add_query;
821 plugin->dhtlog_api->update_trial = &update_trials;
822 plugin->dhtlog_api->insert_route = &add_route;
823 plugin->dhtlog_api->insert_node = &add_node;
824 plugin->dhtlog_api->insert_dhtkey = &add_dhtkey;
825 plugin->dhtlog_api->update_connections = &add_connections;
826 plugin->dhtlog_api->insert_topology = &add_topology;
827 plugin->dhtlog_api->insert_extended_topology = &add_extended_topology;
828 plugin->dhtlog_api->update_topology = &update_topology;
829 plugin->dhtlog_api->set_malicious = &set_malicious;
830 plugin->dhtlog_api->add_generic_stat = &add_generic_stat;
836 * Shutdown the plugin.
839 libgnunet_plugin_dhtlog_mysql_dump_load_done (void * cls)
841 struct GNUNET_DHTLOG_Handle *dhtlog_api = cls;
842 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
843 "MySQL DHT Logger: database shutdown\n");
844 GNUNET_assert(dhtlog_api != NULL);
847 fclose(node_outfile);
848 fclose(query_outfile);
849 fclose(route_outfile);
850 fclose(stat_outfile);
851 fclose(generic_stat_outfile);
852 fclose(extended_topology_outfile);
853 GNUNET_free(dhtlog_api);
857 /* end of plugin_dhtlog_mysql_dump_load.c */