2 This file is part of GNUnet.
3 (C) 2006 - 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file src/dht/plugin_dhtlog_mysql_dump_load.c
23 * @brief MySQL logging plugin to record DHT operations to MySQL server,
24 * but write all queries to file instead of the actual server
25 * so that they can be imported later. Since the first attempt
26 * (writing out SQL queries) seemed rather time consuming on insert,
27 * this plugin writes out tab separated values instead.
29 * @author Nathan Evans
35 #include "gnunet_util_lib.h"
39 #define DEBUG_DHTLOG GNUNET_NO
42 * Maximum number of supported parameters for a prepared
43 * statement. Increase if needed.
48 static unsigned long max_varchar_len;
51 * The configuration the DHT service is running with
53 static const struct GNUNET_CONFIGURATION_Handle *cfg;
55 #define DATE_STR_SIZE 50
57 static unsigned int topology_count;
60 * File(s) to dump all sql statements to.
63 FILE *generic_stat_outfile;
69 FILE *extended_topology_outfile;
74 static char date[DATE_STR_SIZE];
79 memset (date, 0, DATE_STR_SIZE);
80 tmptr = localtime (&timetmp);
82 strftime (date, DATE_STR_SIZE, "%Y-%m-%d %H:%M:%S", tmptr);
90 * Records the current topology (number of connections, time, trial)
92 * @param num_connections how many connections are in the topology
94 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
97 add_topology (int num_connections)
101 return GNUNET_SYSERR;
103 ret = fprintf(outfile, "insert into topology (trialuid, date, connections) values (@temp_trial, \"%s\", %d);\n", get_sql_time(), num_connections);
104 ret = fprintf(outfile, "select max(topology_uid) from topology into @temp_topology;\n");
107 return GNUNET_SYSERR;
111 * Records a connection between two peers in the current topology
113 * @param first one side of the connection
114 * @param second other side of the connection
116 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
119 add_extended_topology (const struct GNUNET_PeerIdentity *first, const struct GNUNET_PeerIdentity *second)
123 return GNUNET_SYSERR;
125 ret = fprintf(extended_topology_outfile, "insert into extended_topology (topology_uid, uid_first, uid_second) values (%u, %s,", topology_count, GNUNET_h2s_full(&first->hashPubKey));
126 ret = fprintf(extended_topology_outfile, "%s);\n", GNUNET_h2s_full(&second->hashPubKey));
130 return GNUNET_SYSERR;
135 * Inserts the specified trial into the dhttests.trials table
137 * @param trialuid return the trialuid of the newly inserted trial
138 * @param num_nodes how many nodes are in the trial
139 * @param topology integer representing topology for this trial
140 * @param blacklist_topology integer representing blacklist topology for this trial
141 * @param connect_topology integer representing connect topology for this trial
142 * @param connect_topology_option integer representing connect topology option
143 * @param connect_topology_option_modifier float to modify connect option
144 * @param topology_percentage percentage modifier for certain topologies
145 * @param topology_probability probability modifier for certain topologies
146 * @param puts number of puts to perform
147 * @param gets number of gets to perform
148 * @param concurrent number of concurrent requests
149 * @param settle_time time to wait between creating topology and starting testing
150 * @param num_rounds number of times to repeat the trial
151 * @param malicious_getters number of malicious GET peers in the trial
152 * @param malicious_putters number of malicious PUT peers in the trial
153 * @param malicious_droppers number of malicious DROP peers in the trial
154 * @param malicious_get_frequency how often malicious gets are sent
155 * @param malicious_put_frequency how often malicious puts are sent
156 * @param stop_closest stop forwarding PUTs if closest node found
157 * @param stop_found stop forwarding GETs if data found
158 * @param strict_kademlia test used kademlia routing algorithm
159 * @param gets_succeeded how many gets did the test driver report success on
160 * @param message string to put into DB for this trial
162 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
164 int add_trial (unsigned long long *trialuid, unsigned int num_nodes, unsigned int topology,
165 unsigned int blacklist_topology, unsigned int connect_topology,
166 unsigned int connect_topology_option, float connect_topology_option_modifier,
167 float topology_percentage, float topology_probability,
168 unsigned int puts, unsigned int gets, unsigned int concurrent, unsigned int settle_time,
169 unsigned int num_rounds, unsigned int malicious_getters, unsigned int malicious_putters,
170 unsigned int malicious_droppers, unsigned int malicious_get_frequency,
171 unsigned int malicious_put_frequency, unsigned int stop_closest, unsigned int stop_found,
172 unsigned int strict_kademlia, unsigned int gets_succeeded,
176 if (trialuid != NULL)
179 return GNUNET_SYSERR;
181 ret = fprintf(outfile, "INSERT INTO trials "
182 "(starttime, numnodes, topology,"
183 "blacklist_topology, connect_topology, connect_topology_option,"
184 "connect_topology_option_modifier, topology_percentage, topology_probability,"
186 "concurrent, settle_time, num_rounds, malicious_getters,"
187 "malicious_putters, malicious_droppers, malicious_get_frequency,"
188 "malicious_put_frequency, stop_closest, stop_found, strict_kademlia, "
189 "gets_succeeded, message) "
190 "VALUES (\"%s\", %u, %u, %u, %u, %u, %f, %f, %f, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, \"%s\");\n",
191 get_sql_time(), num_nodes, topology,
192 blacklist_topology, connect_topology,
193 connect_topology_option, connect_topology_option_modifier,
194 topology_percentage, topology_probability,
195 puts, gets, concurrent, settle_time,
196 num_rounds, malicious_getters, malicious_putters,
197 malicious_droppers, malicious_get_frequency, malicious_put_frequency,
198 stop_closest, stop_found, strict_kademlia, gets_succeeded, message);
200 ret = fprintf(outfile, "SELECT MAX( trialuid ) FROM trials into @temp_trial;\n");
204 return GNUNET_SYSERR;
209 * Inserts the specified stats into the dhttests.generic_stats table
211 * @param peer the peer inserting the statistic
212 * @param name the name of the statistic
213 * @param section the section of the statistic
214 * @param value the value of the statistic
216 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
219 add_generic_stat (const struct GNUNET_PeerIdentity *peer,
221 const char *section, uint64_t value)
224 return GNUNET_SYSERR;
227 fprintf(generic_stat_outfile, "TRIALUID\t%s\t%s\t%s\t%llu\n", GNUNET_h2s_full(&peer->hashPubKey), section, name, (unsigned long long)value);
234 * Inserts the specified stats into the dhttests.node_statistics table
236 * @param peer the peer inserting the statistic
237 * @param route_requests route requests seen
238 * @param route_forwards route requests forwarded
239 * @param result_requests route result requests seen
240 * @param client_requests client requests initiated
241 * @param result_forwards route results forwarded
242 * @param gets get requests handled
243 * @param puts put requests handle
244 * @param data_inserts data inserted at this node
245 * @param find_peer_requests find peer requests seen
246 * @param find_peers_started find peer requests initiated at this node
247 * @param gets_started get requests initiated at this node
248 * @param puts_started put requests initiated at this node
249 * @param find_peer_responses_received find peer responses received locally
250 * @param get_responses_received get responses received locally
251 * @param find_peer_responses_sent find peer responses sent from this node
252 * @param get_responses_sent get responses sent from this node
254 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
257 add_stat (const struct GNUNET_PeerIdentity *peer, unsigned int route_requests,
258 unsigned int route_forwards, unsigned int result_requests,
259 unsigned int client_requests, unsigned int result_forwards,
260 unsigned int gets, unsigned int puts,
261 unsigned int data_inserts, unsigned int find_peer_requests,
262 unsigned int find_peers_started, unsigned int gets_started,
263 unsigned int puts_started, unsigned int find_peer_responses_received,
264 unsigned int get_responses_received, unsigned int find_peer_responses_sent,
265 unsigned int get_responses_sent)
269 return GNUNET_SYSERR;
272 ret = fprintf(stat_outfile, "TRIALUID\t%s\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\t%u\n",
273 GNUNET_h2s_full(&peer->hashPubKey),
274 route_requests, route_forwards, result_requests,
275 client_requests, result_forwards, gets, puts,
276 data_inserts, find_peer_requests, find_peers_started,
277 gets_started, puts_started, find_peer_responses_received,
278 get_responses_received, find_peer_responses_sent,
282 return GNUNET_SYSERR;
287 * Inserts the specified dhtkey into the dhttests.dhtkeys table,
288 * stores return value of dhttests.dhtkeys.dhtkeyuid into dhtkeyuid
290 * @param dhtkeyuid return value
291 * @param dhtkey hashcode of key to insert
293 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
296 add_dhtkey (unsigned long long *dhtkeyuid, const GNUNET_HashCode * dhtkey)
299 if (dhtkeyuid != NULL)
302 if (dhtkey_outfile == NULL)
303 return GNUNET_SYSERR;
306 ret = fprintf(dhtkey_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(dhtkey));
310 return GNUNET_SYSERR;
314 * Inserts the specified node into the dhttests.nodes table
316 * @param nodeuid the inserted node uid
317 * @param node the node to insert
319 * @return GNUNET_OK on success, GNUNET_SYSERR on failure
322 add_node (unsigned long long *nodeuid, struct GNUNET_PeerIdentity * node)
326 if ((node == NULL) || (node_outfile == NULL))
327 return GNUNET_SYSERR;
329 ret = fprintf(node_outfile, "TRIALUID\t%s\n", GNUNET_h2s_full(&node->hashPubKey));
333 return GNUNET_SYSERR;
337 * Update dhttests.trials table with current server time as end time
339 * @param trialuid trial to update
340 * @param gets_succeeded how many gets did the testcase report as successful
342 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
345 update_trials (unsigned long long trialuid,
346 unsigned int gets_succeeded)
350 if (trialuid != current_trial)
353 _("Trialuid to update is not equal to current_trial\n"));
358 return GNUNET_SYSERR;
360 ret = fprintf(outfile, "update trials set endtime=\"%s\", gets_succeeded=%u where trialuid = @temp_trial;\n", get_sql_time(), gets_succeeded);
365 return GNUNET_SYSERR;
370 * Update dhttests.nodes table setting the identified
371 * node as a malicious dropper.
373 * @param peer the peer that was set to be malicious
375 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
378 set_malicious (struct GNUNET_PeerIdentity *peer)
383 return GNUNET_SYSERR;
385 ret = fprintf(outfile, "update nodes set malicious_dropper = 1 where trialuid = @temp_trial and nodeid = \"%s\";\n", GNUNET_h2s_full(&peer->hashPubKey));
390 return GNUNET_SYSERR;
395 * Update dhttests.trials table with total connections information
397 * @param trialuid the trialuid to update
398 * @param totalConnections the number of connections
400 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
403 add_connections (unsigned long long trialuid, unsigned int totalConnections)
407 if (trialuid != current_trial)
410 _("Trialuid to update is not equal to current_trial(!)(?)\n"));
414 return GNUNET_SYSERR;
416 ret = fprintf(outfile, "update trials set totalConnections = %u where trialuid = @temp_trial;\n", totalConnections);
421 return GNUNET_SYSERR;
426 * Update dhttests.topology table with total connections information
428 * @param connections the number of connections
430 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
433 update_topology (unsigned int connections)
437 return GNUNET_SYSERR;
439 ret = fprintf(outfile, "update topology set connections = %u where topology_uid = @temp_topology;\n", connections);
444 return GNUNET_SYSERR;
448 * Inserts the specified query into the dhttests.queries table
450 * @param sqlqueruid inserted query uid
451 * @param queryid dht query id
452 * @param type type of the query
453 * @param hops number of hops query traveled
454 * @param succeeded whether or not query was successful
455 * @param node the node the query hit
456 * @param key the key of the query
458 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
461 add_query (unsigned long long *sqlqueryuid, unsigned long long queryid,
462 unsigned int type, unsigned int hops, int succeeded,
463 const struct GNUNET_PeerIdentity * node, const GNUNET_HashCode * key)
467 if ((outfile == NULL) || (node == NULL) || (key == NULL))
468 return GNUNET_SYSERR;
470 if (sqlqueryuid != NULL)
473 ret = fprintf(query_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
474 ret = fprintf(query_outfile, "%s\t%llu\t%u\t%u\t%u\n", GNUNET_h2s_full(&node->hashPubKey), queryid, type, hops, succeeded);
479 return GNUNET_SYSERR;
483 * Inserts the specified route information into the dhttests.routes table
485 * @param sqlqueruid inserted query uid
486 * @param queryid dht query id
487 * @param type type of the query
488 * @param hops number of hops query traveled
489 * @param succeeded whether or not query was successful
490 * @param node the node the query hit
491 * @param key the key of the query
492 * @param from_node the node that sent the message to node
493 * @param to_node next node to forward message to
495 * @return GNUNET_OK on success, GNUNET_SYSERR on failure.
498 add_route (unsigned long long *sqlqueryuid, unsigned long long queryid,
499 unsigned int type, unsigned int hops,
500 int succeeded, const struct GNUNET_PeerIdentity * node,
501 const GNUNET_HashCode * key, const struct GNUNET_PeerIdentity * from_node,
502 const struct GNUNET_PeerIdentity * to_node)
506 if ((outfile == NULL) || (node == NULL) || (key == NULL))
507 return GNUNET_SYSERR;
509 if (sqlqueryuid != NULL)
512 ret = fprintf(route_outfile, "TRIALUID\t%s\t", GNUNET_h2s_full(key));
513 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&node->hashPubKey));
514 if (from_node == NULL)
515 ret = fprintf(route_outfile, "0\t");
517 ret = fprintf(route_outfile, "%s\t", GNUNET_h2s_full(&from_node->hashPubKey));
520 ret = fprintf(route_outfile, "0\t%llu\t%u\t%u\t%d\n", queryid, type, hops, succeeded);
522 ret = fprintf(route_outfile, "%s\t%llu\t%u\t%u\t%d\n", GNUNET_h2s_full(&to_node->hashPubKey), queryid, type, hops, succeeded);
527 return GNUNET_SYSERR;
531 * Provides the dhtlog api
533 * @param c the configuration to use to connect to a server
535 * @return the handle to the server, or NULL on error
538 libgnunet_plugin_dhtlog_mysql_dump_load_init (void * cls)
540 struct GNUNET_DHTLOG_Plugin *plugin = cls;
547 max_varchar_len = 255;
549 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MySQL (DUMP) DHT Logger: initializing\n");
551 if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (plugin->cfg,
555 outfile_path = GNUNET_strdup("");
558 GNUNET_asprintf (&outfile_name,
564 fn = GNUNET_STRINGS_filename_expand (outfile_name);
566 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
567 outfile = FOPEN (fn, "w");
571 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
573 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
574 _("Failed to create or access directory for log file `%s'\n"),
576 GNUNET_free(outfile_path);
577 GNUNET_free(outfile_name);
582 GNUNET_asprintf (&outfile_name,
588 fn = GNUNET_STRINGS_filename_expand (outfile_name);
590 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
591 node_outfile = FOPEN (fn, "w");
593 if (node_outfile == NULL)
595 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
597 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
598 _("Failed to create or access directory for log file `%s'\n"),
600 GNUNET_free(outfile_path);
601 GNUNET_free(outfile_name);
606 GNUNET_asprintf (&outfile_name,
612 fn = GNUNET_STRINGS_filename_expand (outfile_name);
614 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
615 route_outfile = FOPEN (fn, "w");
617 if (route_outfile == NULL)
619 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
621 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
622 _("Failed to create or access directory for log file `%s'\n"),
624 GNUNET_free(outfile_path);
625 GNUNET_free(outfile_name);
630 GNUNET_asprintf (&outfile_name,
636 fn = GNUNET_STRINGS_filename_expand (outfile_name);
638 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
639 query_outfile = FOPEN (fn, "w");
641 if (query_outfile == NULL)
643 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
645 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
646 _("Failed to create or access directory for log file `%s'\n"),
648 GNUNET_free(outfile_path);
649 GNUNET_free(outfile_name);
654 GNUNET_asprintf (&outfile_name,
660 fn = GNUNET_STRINGS_filename_expand (outfile_name);
662 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
663 stat_outfile = FOPEN (fn, "w");
665 if (stat_outfile == NULL)
667 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
669 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
670 _("Failed to create or access directory for log file `%s'\n"),
672 GNUNET_free(outfile_path);
673 GNUNET_free(outfile_name);
678 GNUNET_asprintf (&outfile_name,
681 "mysqldump_generic_stats",
684 fn = GNUNET_STRINGS_filename_expand (outfile_name);
686 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
687 generic_stat_outfile = FOPEN (fn, "w");
689 if (generic_stat_outfile == NULL)
691 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
693 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
694 _("Failed to create or access directory for log file `%s'\n"),
696 GNUNET_free(outfile_path);
697 GNUNET_free(outfile_name);
702 GNUNET_asprintf (&outfile_name,
708 fn = GNUNET_STRINGS_filename_expand (outfile_name);
710 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
711 dhtkey_outfile = FOPEN (fn, "w");
713 if (dhtkey_outfile == NULL)
715 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
717 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
718 _("Failed to create or access directory for log file `%s'\n"),
720 GNUNET_free(outfile_path);
721 GNUNET_free(outfile_name);
726 GNUNET_asprintf (&outfile_name,
729 "mysqldump_extended_topology",
732 fn = GNUNET_STRINGS_filename_expand (outfile_name);
734 dirwarn = (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn));
735 extended_topology_outfile = FOPEN (fn, "w");
737 if (extended_topology_outfile == NULL)
739 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "fopen", fn);
741 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
742 _("Failed to create or access directory for log file `%s'\n"),
744 GNUNET_free(outfile_path);
745 GNUNET_free(outfile_name);
750 GNUNET_free (outfile_path);
751 GNUNET_free (outfile_name);
754 GNUNET_assert(plugin->dhtlog_api == NULL);
755 plugin->dhtlog_api = GNUNET_malloc(sizeof(struct GNUNET_DHTLOG_Handle));
756 plugin->dhtlog_api->insert_trial = &add_trial;
757 plugin->dhtlog_api->insert_stat = &add_stat;
758 plugin->dhtlog_api->insert_query = &add_query;
759 plugin->dhtlog_api->update_trial = &update_trials;
760 plugin->dhtlog_api->insert_route = &add_route;
761 plugin->dhtlog_api->insert_node = &add_node;
762 plugin->dhtlog_api->insert_dhtkey = &add_dhtkey;
763 plugin->dhtlog_api->update_connections = &add_connections;
764 plugin->dhtlog_api->insert_topology = &add_topology;
765 plugin->dhtlog_api->insert_extended_topology = &add_extended_topology;
766 plugin->dhtlog_api->update_topology = &update_topology;
767 plugin->dhtlog_api->set_malicious = &set_malicious;
768 plugin->dhtlog_api->add_generic_stat = &add_generic_stat;
774 * Shutdown the plugin.
777 libgnunet_plugin_dhtlog_mysql_dump_load_done (void * cls)
779 struct GNUNET_DHTLOG_Handle *dhtlog_api = cls;
780 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
781 "MySQL DHT Logger: database shutdown\n");
782 GNUNET_assert(dhtlog_api != NULL);
785 fclose(node_outfile);
786 fclose(query_outfile);
787 fclose(route_outfile);
788 fclose(stat_outfile);
789 fclose(generic_stat_outfile);
790 fclose(extended_topology_outfile);
791 GNUNET_free(dhtlog_api);
795 /* end of plugin_dhtlog_mysql_dump_load.c */