2 This file is part of GNUnet.
3 (C) 2006 - 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file src/dht/plugin_dhtlog_mysql_dump_load.c
23 * @brief MySQL logging plugin to record DHT operations to MySQL server,
24 * but write all queries to file instead of the actual server
25 * so that they can be imported later. Since the first attempt
26 * (writing out SQL queries) seemed rather time consuming on insert,
27 * this plugin writes out tab separated values instead.
29 * @author Nathan Evans
35 #include "gnunet_util_lib.h"
39 #define DEBUG_DHTLOG GNUNET_YES
42 * Maximum number of supported parameters for a prepared
43 * statement. Increase if needed.
48 static unsigned long max_varchar_len;
51 * The configuration the DHT service is running with
53 static const struct GNUNET_CONFIGURATION_Handle *cfg;
55 #define DATE_STR_SIZE 50
57 static unsigned int topology_count;
60 * File(s) to dump all sql statements to.
63 FILE *generic_stat_outfile;
69 FILE *extended_topology_outfile;
74 static char date[DATE_STR_SIZE];
79 memset (date, 0, DATE_STR_SIZE);
80 tmptr = localtime (&timetmp);
82 strftime (date, DATE_STR_SIZE, "%Y-%m-%d %H:%M:%S", tmptr);
90 * Records the current topology (number of connections, time, trial)
92 * @param num_connections how many connections are in the topology
94 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
97 add_topology (int num_connections)
101 return GNUNET_SYSERR;
103 ret = fprintf(outfile, "insert into topology (trialuid, date, connections) values (@temp_trial, \"%s\", %d);\n", get_sql_time(), num_connections);
105 return GNUNET_SYSERR;
106 ret = fprintf(outfile, "select max(topology_uid) from topology into @temp_topology;\n");
109 return GNUNET_SYSERR;
113 * Inserts the specified round into the dhttests.rounds table
115 * @param round_type the type of round that is being started
116 * @param round_count counter for the round (if applicable)
118 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
120 int add_round (unsigned int round_type, unsigned int round_count)
124 return GNUNET_SYSERR;
126 ret = fprintf(outfile, "insert into rounds (trialuid, round_type, round_count, starttime) values (@temp_trial, \"%u\", \"%u\", \"%s\");\n", round_type, round_count, get_sql_time());
130 return GNUNET_SYSERR;
135 * Records a connection between two peers in the current topology
137 * @param first one side of the connection
138 * @param second other side of the connection
140 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
143 add_extended_topology (const struct GNUNET_PeerIdentity *first, const struct GNUNET_PeerIdentity *second)
147 return GNUNET_SYSERR;
149 ret = fprintf(extended_topology_outfile, "insert into extended_topology (topology_uid, uid_first, uid_second) values (%u, %s,", topology_count, GNUNET_h2s_full(&first->hashPubKey));
151 return GNUNET_SYSERR;
152 ret = fprintf(extended_topology_outfile, "%s);\n", GNUNET_h2s_full(&second->hashPubKey));
156 return GNUNET_SYSERR;
161 * Inserts the specified trial into the dhttests.trials table
163 * @param trial_info struct containing the data to insert about this trial
165 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
167 int add_trial (struct GNUNET_DHTLOG_TrialInfo *trial_info)
170 trial_info->trialuid = 0;
172 return GNUNET_SYSERR;
174 ret = fprintf(outfile, "INSERT INTO trials "
175 "(starttime, other_trial_identifier, numnodes, topology,"
176 "blacklist_topology, connect_topology, connect_topology_option,"
177 "connect_topology_option_modifier, topology_percentage, topology_probability,"
179 "concurrent, settle_time, num_rounds, malicious_getters,"
180 "malicious_putters, malicious_droppers, malicious_get_frequency,"
181 "malicious_put_frequency, stop_closest, stop_found, strict_kademlia, "
182 "gets_succeeded, message) "
183 "VALUES (\"%s\", %u, %u, %u, %u, %u, %u, %f, %f, %f, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, \"%s\");\n",
184 get_sql_time(), trial_info->other_identifier, trial_info->num_nodes, trial_info->topology,
185 trial_info->blacklist_topology, trial_info->connect_topology,
186 trial_info->connect_topology_option, trial_info->connect_topology_option_modifier,
187 trial_info->topology_percentage, trial_info->topology_probability,
188 trial_info->puts, trial_info->gets, trial_info->concurrent, trial_info->settle_time,
189 trial_info->num_rounds, trial_info->malicious_getters, trial_info->malicious_putters,
190 trial_info->malicious_droppers, trial_info->malicious_get_frequency, trial_info->malicious_put_frequency,
191 trial_info->stop_closest, trial_info->stop_found, trial_info->strict_kademlia, trial_info->gets_succeeded, trial_info->message);
194 return GNUNET_SYSERR;
196 ret = fprintf(outfile, "SELECT MAX( trialuid ) FROM trials into @temp_trial;\n");
201 return GNUNET_SYSERR;
206 * Inserts the specified stats into the dhttests.generic_stats table
208 * @param peer the peer inserting the statistic
209 * @param name the name of the statistic
210 * @param section the section of the statistic
211 * @param value the value of the statistic
213 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
216 add_generic_stat (const struct GNUNET_PeerIdentity *peer,
218 const char *section, uint64_t value)
221 return GNUNET_SYSERR;
224 fprintf(generic_stat_outfile, "TRIALUID\t%s\t%s\t%s\t%llu\n", GNUNET_h2s_full(&peer->hashPubKey), section, name, (unsigned long long)value);
231 * Inserts the specified stats into the dhttests.node_statistics table
233 * @param peer the peer inserting the statistic
234 * @param route_requests route requests seen
235 * @param route_forwards route requests forwarded
236 * @param result_requests route result requests seen
237 * @param client_requests client requests initiated
238 * @param result_forwards route results forwarded
239 * @param gets get requests handled
240 * @param puts put requests handle
241 * @param data_inserts data inserted at this node
242 * @param find_peer_requests find peer requests seen
243 * @param find_peers_started find peer requests initiated at this node
244 * @param gets_started get requests initiated at this node
245 * @param puts_started put requests initiated at this node
246 * @param find_peer_responses_received find peer responses received locally
247 * @param get_responses_received get responses received locally
248 * @param find_peer_responses_sent find peer responses sent from this node
249 * @param get_responses_sent get responses sent from this node
251 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
254 add_stat (const struct GNUNET_PeerIdentity *peer, unsigned int route_requests,
255 unsigned int route_forwards, unsigned int result_requests,
256 unsigned int client_requests, unsigned int result_forwards,
257 unsigned int gets, unsigned int puts,
258 unsigned int data_inserts, unsigned int find_peer_requests,
259 unsigned int find_peers_started, unsigned int gets_started,
260 unsigned int puts_started, unsigned int find_peer_responses_received,
261 unsigned int get_responses_received, unsigned int find_peer_responses_sent,
262 unsigned int get_responses_sent)
266 return GNUNET_SYSERR;
269 ret = fprintf(stat_outfile, "TRIALUID\t%s\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\n",
270 GNUNET_h2s_full(&peer->hashPubKey),
271 route_requests, route_forwards, result_requests,
272 client_requests, result_forwards, gets, puts,
273 data_inserts, find_peer_requests, find_peers_started,
274 gets_started, puts_started, find_peer_responses_received,
275 get_responses_received, find_peer_responses_sent,
279 return GNUNET_SYSERR;
284 return GNUNET_SYSERR;
287 * Inserts the specified dhtkey into the dhttests.dhtkeys table,
288 * stores return value of dhttests.dhtkeys.dhtkeyuid into dhtkeyuid
290 * @param dhtkeyuid return value
291 * @param dhtkey hashcode of key to insert
293 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
296 add_dhtkey (unsigned long long *dhtkeyuid, const GNUNET_HashCode * dhtkey)
299 if (dhtkeyuid != NULL)
302 if ((dhtkey_outfile == NULL) || (dhtkey == NULL))
303 return GNUNET_SYSERR;
305 ret = fprintf(dhtkey_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(dhtkey));
310 return GNUNET_SYSERR;
314 * Inserts the specified node into the dhttests.nodes table
316 * @param nodeuid the inserted node uid
317 * @param node the node to insert
319 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
322 add_node (unsigned long long *nodeuid, struct GNUNET_PeerIdentity * node)
326 if ((node == NULL) || (node_outfile == NULL))
327 return GNUNET_SYSERR;
329 ret = fprintf(node_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(&node->hashPubKey));
333 return GNUNET_SYSERR;
337 * Update dhttests.trials table with current server time as end time
339 * @param trialuid trial to update
340 * @param gets_succeeded how many gets did the testcase report as successful
342 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
345 update_trials (unsigned long long trialuid,
346 unsigned int gets_succeeded)
350 /* if (trialuid != current_trial)
353 _("Trialuid to update is not equal to current_trial\n"));
358 return GNUNET_SYSERR;
360 ret = fprintf(outfile, "update trials set endtime=\"%s\", gets_succeeded=%u where trialuid = @temp_trial;\n", get_sql_time(), gets_succeeded);
365 return GNUNET_SYSERR;
370 * Update dhttests.nodes table setting the identified
371 * node as a malicious dropper.
373 * @param peer the peer that was set to be malicious
375 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
378 set_malicious (struct GNUNET_PeerIdentity *peer)
383 return GNUNET_SYSERR;
385 ret = fprintf(outfile, "update nodes set malicious_dropper = 1 where trialuid = @temp_trial and nodeid = \"%s\";\n", GNUNET_h2s_full(&peer->hashPubKey));
390 return GNUNET_SYSERR;
395 * Update dhttests.trials table with total connections information
397 * @param trialuid the trialuid to update
398 * @param totalConnections the number of connections
400 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
403 add_connections (unsigned long long trialuid, unsigned int totalConnections)
407 /* if (trialuid != current_trial)
410 _("Trialuid to update is not equal to current_trial(!)(?)\n"));
414 return GNUNET_SYSERR;
416 ret = fprintf(outfile, "update trials set totalConnections = %u where trialuid = @temp_trial;\n", totalConnections);
421 return GNUNET_SYSERR;
426 * Update dhttests.topology table with total connections information
428 * @param connections the number of connections
430 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
433 update_topology (unsigned int connections)
437 return GNUNET_SYSERR;
439 ret = fprintf(outfile, "update topology set connections = %u where topology_uid = @temp_topology;\n", connections);
444 return GNUNET_SYSERR;
448 * Inserts the specified query into the dhttests.queries table
450 * @param sqlqueruid inserted query uid
451 * @param queryid dht query id
452 * @param type type of the query
453 * @param hops number of hops query traveled
454 * @param succeeded whether or not query was successful
455 * @param node the node the query hit
456 * @param key the key of the query
458 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
461 add_query (unsigned long long *sqlqueryuid, unsigned long long queryid,
462 unsigned int type, unsigned int hops, int succeeded,
463 const struct GNUNET_PeerIdentity * node, const GNUNET_HashCode * key)
467 if ((outfile == NULL) || (node == NULL) || (key == NULL))
468 return GNUNET_SYSERR;
470 if (sqlqueryuid != NULL)
473 ret = fprintf(query_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
476 return GNUNET_SYSERR;
478 ret = fprintf(query_outfile, "%s\t%llu\t%u\t%u\t%u\t%s\n", GNUNET_h2s_full(&node->hashPubKey), queryid, type, hops, succeeded, get_sql_time());
483 return GNUNET_SYSERR;
487 * Inserts the specified route information into the dhttests.routes table
489 * @param sqlqueruid inserted query uid
490 * @param queryid dht query id
491 * @param type type of the query
492 * @param hops number of hops query traveled
493 * @param succeeded whether or not query was successful
494 * @param node the node the query hit
495 * @param key the key of the query
496 * @param from_node the node that sent the message to node
497 * @param to_node next node to forward message to
499 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
502 add_route (unsigned long long *sqlqueryuid, unsigned long long queryid,
503 unsigned int type, unsigned int hops,
504 int succeeded, const struct GNUNET_PeerIdentity * node,
505 const GNUNET_HashCode * key, const struct GNUNET_PeerIdentity * from_node,
506 const struct GNUNET_PeerIdentity * to_node)
510 if ((outfile == NULL) || (node == NULL) || (key == NULL))
511 return GNUNET_SYSERR;
513 if (sqlqueryuid != NULL)
516 ret = fprintf(route_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
518 return GNUNET_SYSERR;
520 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&node->hashPubKey));
522 return GNUNET_SYSERR;
523 if (from_node == NULL)
524 ret = fprintf(route_outfile, "0\t");
526 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&from_node->hashPubKey));
528 return GNUNET_SYSERR;
531 ret = fprintf(route_outfile, "0\t%llu\t%u\t%u\t%d\n", queryid, type, hops, succeeded);
533 ret = fprintf(route_outfile, "%s\t%llu\t%u\t%u\t%d\n", GNUNET_h2s_full(&to_node->hashPubKey), queryid, type, hops, succeeded);
538 return GNUNET_SYSERR;
542 * Provides the dhtlog api
544 * @param c the configuration to use to connect to a server
546 * @return the handle to the server, or NULL on error
549 libgnunet_plugin_dhtlog_mysql_dump_load_init (void * cls)
551 struct GNUNET_DHTLOG_Plugin *plugin = cls;
558 max_varchar_len = 255;
560 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MySQL (DUMP) DHT Logger: initializing\n");
562 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (plugin->cfg,
566 outfile_path = GNUNET_strdup("");
569 GNUNET_asprintf (&outfile_name,
575 fn = GNUNET_STRINGS_filename_expand (outfile_name);
579 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
580 GNUNET_free(outfile_path);
581 GNUNET_free(outfile_name);
585 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
586 outfile = FOPEN (fn, "w");
590 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
592 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
593 _("Failed to create or access directory for log file `%s'\n"),
595 GNUNET_free(outfile_path);
596 GNUNET_free(outfile_name);
601 GNUNET_asprintf (&outfile_name,
607 fn = GNUNET_STRINGS_filename_expand (outfile_name);
611 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
612 GNUNET_free(outfile_path);
613 GNUNET_free(outfile_name);
617 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
618 node_outfile = FOPEN (fn, "w");
620 if (node_outfile == NULL)
622 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
624 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
625 _("Failed to create or access directory for log file `%s'\n"),
627 GNUNET_free(outfile_path);
628 GNUNET_free(outfile_name);
633 GNUNET_asprintf (&outfile_name,
639 fn = GNUNET_STRINGS_filename_expand (outfile_name);
643 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
644 GNUNET_free(outfile_path);
645 GNUNET_free(outfile_name);
649 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
650 route_outfile = FOPEN (fn, "w");
652 if (route_outfile == NULL)
654 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
656 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
657 _("Failed to create or access directory for log file `%s'\n"),
659 GNUNET_free(outfile_path);
660 GNUNET_free(outfile_name);
665 GNUNET_asprintf (&outfile_name,
671 fn = GNUNET_STRINGS_filename_expand (outfile_name);
675 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
676 GNUNET_free(outfile_path);
677 GNUNET_free(outfile_name);
681 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
682 query_outfile = FOPEN (fn, "w");
684 if (query_outfile == NULL)
686 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
688 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
689 _("Failed to create or access directory for log file `%s'\n"),
691 GNUNET_free(outfile_path);
692 GNUNET_free(outfile_name);
697 GNUNET_asprintf (&outfile_name,
703 fn = GNUNET_STRINGS_filename_expand (outfile_name);
707 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
708 GNUNET_free(outfile_path);
709 GNUNET_free(outfile_name);
713 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
714 stat_outfile = FOPEN (fn, "w");
716 if (stat_outfile == NULL)
718 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
720 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
721 _("Failed to create or access directory for log file `%s'\n"),
723 GNUNET_free(outfile_path);
724 GNUNET_free(outfile_name);
729 GNUNET_asprintf (&outfile_name,
732 "mysqldump_generic_stats",
735 fn = GNUNET_STRINGS_filename_expand (outfile_name);
739 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
740 GNUNET_free(outfile_path);
741 GNUNET_free(outfile_name);
745 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
746 generic_stat_outfile = FOPEN (fn, "w");
748 if (generic_stat_outfile == NULL)
750 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
752 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
753 _("Failed to create or access directory for log file `%s'\n"),
755 GNUNET_free(outfile_path);
756 GNUNET_free(outfile_name);
761 GNUNET_asprintf (&outfile_name,
767 fn = GNUNET_STRINGS_filename_expand (outfile_name);
771 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
772 GNUNET_free(outfile_path);
773 GNUNET_free(outfile_name);
777 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
778 dhtkey_outfile = FOPEN (fn, "w");
780 if (dhtkey_outfile == NULL)
782 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
784 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
785 _("Failed to create or access directory for log file `%s'\n"),
787 GNUNET_free(outfile_path);
788 GNUNET_free(outfile_name);
793 GNUNET_asprintf (&outfile_name,
796 "mysqldump_extended_topology",
799 fn = GNUNET_STRINGS_filename_expand (outfile_name);
803 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, _("Failed to get full path for `%s'\n"), outfile_name);
804 GNUNET_free(outfile_path);
805 GNUNET_free(outfile_name);
809 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
810 extended_topology_outfile = FOPEN (fn, "w");
812 if (extended_topology_outfile == NULL)
814 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
816 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
817 _("Failed to create or access directory for log file `%s'\n"),
819 GNUNET_free(outfile_path);
820 GNUNET_free(outfile_name);
825 GNUNET_free (outfile_path);
826 GNUNET_free (outfile_name);
829 GNUNET_assert(plugin->dhtlog_api == NULL);
830 plugin->dhtlog_api = GNUNET_malloc(sizeof(struct GNUNET_DHTLOG_Handle));
831 plugin->dhtlog_api->insert_trial = &add_trial;
832 plugin->dhtlog_api->insert_stat = &add_stat;
833 plugin->dhtlog_api->insert_round = &add_round;
834 plugin->dhtlog_api->insert_query = &add_query;
835 plugin->dhtlog_api->update_trial = &update_trials;
836 plugin->dhtlog_api->insert_route = &add_route;
837 plugin->dhtlog_api->insert_node = &add_node;
838 plugin->dhtlog_api->insert_dhtkey = &add_dhtkey;
839 plugin->dhtlog_api->update_connections = &add_connections;
840 plugin->dhtlog_api->insert_topology = &add_topology;
841 plugin->dhtlog_api->insert_extended_topology = &add_extended_topology;
842 plugin->dhtlog_api->update_topology = &update_topology;
843 plugin->dhtlog_api->set_malicious = &set_malicious;
844 plugin->dhtlog_api->add_generic_stat = &add_generic_stat;
850 * Shutdown the plugin.
853 libgnunet_plugin_dhtlog_mysql_dump_load_done (void * cls)
855 struct GNUNET_DHTLOG_Handle *dhtlog_api = cls;
856 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
857 "MySQL DHT Logger: database shutdown\n");
858 GNUNET_assert(dhtlog_api != NULL);
861 fclose(node_outfile);
862 fclose(query_outfile);
863 fclose(route_outfile);
864 fclose(stat_outfile);
865 fclose(generic_stat_outfile);
866 fclose(extended_topology_outfile);
867 GNUNET_free(dhtlog_api);
871 /* end of plugin_dhtlog_mysql_dump_load.c */