2 This file is part of GNUnet.
3 Copyright (C) 2011, 2012 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
23 * @file regex/gnunet-regex-simulation-profiler.c
24 * @brief Regex profiler that dumps all DFAs into a database instead of
25 * using the DHT (with cadet).
26 * @author Maximilian Szengel
27 * @author Christophe Genevey
32 #include "gnunet_util_lib.h"
33 #include "regex_internal_lib.h"
34 #include "gnunet_mysql_lib.h"
35 #include "gnunet_my_lib.h"
36 #include <mysql/mysql.h>
39 * MySQL statement to insert an edge.
41 #define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` "\
42 "(`key`, `label`, `to_key`, `accepting`) "\
43 "VALUES (?, ?, ?, ?);"
46 * MySQL statement to select a key count.
48 #define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` "\
49 "WHERE `key` = ? AND `label` = ?;"
52 * Simple struct to keep track of progress, and print a
53 * nice little percentage meter for long running tasks.
58 * Total number of elements.
63 * Intervall for printing percentage.
68 * Number of dots to print.
75 unsigned int completed;
78 * Should the meter be printed?
83 * String to print on startup.
90 * Handle for the progress meter
92 static struct ProgressMeter *meter;
95 * Scan task identifier;
97 static struct GNUNET_SCHEDULER_Task *scan_task;
100 * Global testing status.
107 static struct GNUNET_MYSQL_Context *mysql_ctx;
110 * MySQL prepared statement handle.
112 static struct GNUNET_MYSQL_StatementHandle *stmt_handle;
115 * MySQL prepared statement handle for `key` select.
117 static struct GNUNET_MYSQL_StatementHandle *select_stmt_handle;
122 static char *table_name;
125 * Policy dir containing files that contain policies.
127 static char *policy_dir;
130 * Number of policy files.
132 static unsigned int num_policy_files;
135 * Number of policies.
137 static unsigned int num_policies;
140 * Maximal path compression length.
142 static unsigned int max_path_compression;
145 * Number of merged transitions.
147 static unsigned long long num_merged_transitions;
150 * Number of merged states from different policies.
152 static unsigned long long num_merged_states;
155 * Prefix to add before every regex we're announcing.
157 static char *regex_prefix;
161 * Create a meter to keep track of the progress of some task.
163 * @param total the total number of items to complete
164 * @param start_string a string to prefix the meter with (if printing)
165 * @param print GNUNET_YES to print the meter, GNUNET_NO to count
168 * @return the progress meter
170 static struct ProgressMeter *
171 create_meter (unsigned int total, char *start_string, int print)
173 struct ProgressMeter *ret;
175 ret = GNUNET_new (struct ProgressMeter);
178 ret->modnum = total / 4;
179 if (ret->modnum == 0) /* Divide by zero check */
181 ret->dotnum = (total / 50) + 1;
182 if (start_string != NULL)
183 ret->startup_string = GNUNET_strdup (start_string);
185 ret->startup_string = GNUNET_strdup ("");
192 * Update progress meter (increment by one).
194 * @param meter the meter to update and print info for
196 * @return GNUNET_YES if called the total requested,
197 * GNUNET_NO if more items expected
200 update_meter (struct ProgressMeter *meter)
202 if (meter->print == GNUNET_YES)
204 if (meter->completed % meter->modnum == 0)
206 if (meter->completed == 0)
208 FPRINTF (stdout, "%sProgress: [0%%", meter->startup_string);
211 FPRINTF (stdout, "%d%%",
212 (int) (((float) meter->completed / meter->total) * 100));
214 else if (meter->completed % meter->dotnum == 0)
215 FPRINTF (stdout, "%s", ".");
217 if (meter->completed + 1 == meter->total)
218 FPRINTF (stdout, "%d%%]\n", 100);
223 if (meter->completed == meter->total)
225 if (meter->completed > meter->total)
226 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
232 * Reset progress meter.
234 * @param meter the meter to reset
236 * @return #GNUNET_YES if meter reset,
237 * #GNUNET_SYSERR on error
240 reset_meter (struct ProgressMeter *meter)
243 return GNUNET_SYSERR;
245 meter->completed = 0;
251 * Release resources for meter
253 * @param meter the meter to free
256 free_meter (struct ProgressMeter *meter)
258 GNUNET_free_non_null (meter->startup_string);
269 do_shutdown (void *cls)
271 if (NULL != mysql_ctx)
273 GNUNET_MYSQL_context_destroy (mysql_ctx);
285 * Abort task to run on test timed out.
287 * FIXME: this doesn't actually work, it used to cancel
288 * the already running 'scan_task', but now that should
289 * always be NULL and do nothing. We instead need to set
290 * a global variable and abort scan_task internally, not
298 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
299 if (NULL != scan_task)
301 GNUNET_SCHEDULER_cancel (scan_task);
304 result = GNUNET_SYSERR;
305 GNUNET_SCHEDULER_shutdown ();
309 * Iterator over all states that inserts each state into the MySQL db.
311 * @param cls closure.
312 * @param key hash for current state.
313 * @param proof proof for current state.
314 * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not.
315 * @param num_edges number of edges leaving current state.
316 * @param edges edges leaving current state.
319 regex_iterator (void *cls,
320 const struct GNUNET_HashCode *key,
323 unsigned int num_edges,
324 const struct REGEX_BLOCK_Edge *edges)
329 uint32_t iaccepting = (uint32_t)accepting;
332 GNUNET_assert (NULL != mysql_ctx);
334 for (i = 0; i < num_edges; i++)
336 struct GNUNET_MY_QueryParam params_select[] = {
337 GNUNET_MY_query_param_auto_from_type (key),
338 GNUNET_MY_query_param_string (edges[i].label),
339 GNUNET_MY_query_param_end
342 struct GNUNET_MY_ResultSpec results_select[] = {
343 GNUNET_MY_result_spec_uint64 (&total),
344 GNUNET_MY_result_spec_end
348 GNUNET_MY_exec_prepared (mysql_ctx,
352 if (GNUNET_SYSERR == result)
354 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
355 "Error executing prepared mysql select statement\n");
356 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
361 GNUNET_MY_extract_result (select_stmt_handle,
364 if (GNUNET_SYSERR == result)
366 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
367 "Error extracting result mysql select statement\n");
368 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
372 if (-1 != total && total > 0)
374 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
375 (unsigned long long)total,
376 GNUNET_h2s (key), edges[i].label);
379 struct GNUNET_MY_QueryParam params_stmt[] = {
380 GNUNET_MY_query_param_auto_from_type (&key),
381 GNUNET_MY_query_param_string (edges[i].label),
382 GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
383 GNUNET_MY_query_param_uint32 (&iaccepting),
384 GNUNET_MY_query_param_end
388 GNUNET_MY_exec_prepared (mysql_ctx,
394 char *key_str = GNUNET_strdup (GNUNET_h2s (key));
395 char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
397 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
403 GNUNET_free (key_str);
404 GNUNET_free (to_key_str);
405 num_merged_transitions++;
407 else if (-1 != total)
412 if (GNUNET_SYSERR == result || (1 != result && 0 != result))
414 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
415 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
417 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
423 struct GNUNET_MY_QueryParam params_stmt[] = {
424 GNUNET_MY_query_param_auto_from_type (key),
425 GNUNET_MY_query_param_string (""),
426 GNUNET_MY_query_param_auto_from_type (NULL),
427 GNUNET_MY_query_param_uint32 (&iaccepting),
428 GNUNET_MY_query_param_end
432 GNUNET_MY_exec_prepared (mysql_ctx,
436 if (1 != result && 0 != result)
438 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
439 "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
441 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
448 * Announce a regex by creating the DFA and iterating over each state, inserting
449 * each state into a MySQL database.
451 * @param regex regular expression.
452 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure.
455 announce_regex (const char *regex)
457 struct REGEX_INTERNAL_Automaton *dfa;
460 REGEX_INTERNAL_construct_dfa (regex,
462 max_path_compression);
466 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
467 "Failed to create DFA for regex %s\n",
469 GNUNET_SCHEDULER_add_now (&do_abort, NULL);
470 return GNUNET_SYSERR;
472 REGEX_INTERNAL_iterate_all_edges (dfa,
473 ®ex_iterator, NULL);
474 REGEX_INTERNAL_automaton_destroy (dfa);
481 * Function called with a filename.
484 * @param filename complete filename (absolute path)
485 * @return #GNUNET_OK to continue to iterate,
486 * #GNUNET_SYSERR to abort iteration with error!
489 policy_filename_cb (void *cls, const char *filename)
497 GNUNET_assert (NULL != filename);
499 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
500 "Announcing regexes from file %s\n",
503 if (GNUNET_YES != GNUNET_DISK_file_test (filename))
505 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
506 "Could not find policy file %s\n",
511 GNUNET_DISK_file_size (filename, &filesize,
512 GNUNET_YES, GNUNET_YES))
516 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
520 data = GNUNET_malloc (filesize);
521 if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
524 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
525 "Could not read policy file %s.\n",
530 update_meter (meter);
535 while (offset < (filesize - 1))
538 if (((data[offset] == '\n')) && (buf != &data[offset]))
542 buf = &data[offset + 1];
544 else if ((data[offset] == '\n') || (data[offset] == '\0'))
545 buf = &data[offset + 1];
548 GNUNET_asprintf (®ex, "%s(%s)", regex_prefix, data);
549 GNUNET_assert (NULL != regex);
550 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
551 "Announcing regex: %s\n", regex);
553 if (GNUNET_OK != announce_regex (regex))
555 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
556 "Could not announce regex %s\n",
566 * Iterate over files contained in policy_dir.
571 do_directory_scan (void *cls)
573 struct GNUNET_TIME_Absolute start_time;
574 struct GNUNET_TIME_Relative duration;
577 /* Create an MySQL prepared statement for the inserts */
579 GNUNET_asprintf (&stmt, INSERT_EDGE_STMT, table_name);
580 stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
583 GNUNET_asprintf (&stmt, SELECT_KEY_STMT, table_name);
584 select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
587 GNUNET_assert (NULL != stmt_handle);
589 meter = create_meter (num_policy_files,
590 "Announcing policy files\n",
592 start_time = GNUNET_TIME_absolute_get ();
593 GNUNET_DISK_directory_scan (policy_dir,
596 duration = GNUNET_TIME_absolute_get_duration (start_time);
601 printf ("Announced %u files containing %u policies in %s\n"
602 "Duplicate transitions: %llu\nMerged states: %llu\n",
605 GNUNET_STRINGS_relative_time_to_string (duration, GNUNET_NO),
606 num_merged_transitions,
609 GNUNET_SCHEDULER_shutdown ();
614 * Main function that will be run by the scheduler.
617 * @param args remaining command-line arguments
618 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
619 * @param config configuration
625 const struct GNUNET_CONFIGURATION_Handle *config)
630 _("No policy directory specified on command line. Exiting.\n"));
631 result = GNUNET_SYSERR;
635 GNUNET_DISK_directory_test (args[0], GNUNET_YES))
638 _("Specified policies directory does not exist. Exiting.\n"));
639 result = GNUNET_SYSERR;
642 policy_dir = args[0];
644 num_policy_files = GNUNET_DISK_directory_scan (policy_dir,
648 if (NULL == table_name)
650 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
651 "No table name specified, using default \"NFA\".\n");
655 mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
656 if (NULL == mysql_ctx)
658 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
659 "Failed to create mysql context\n");
660 result = GNUNET_SYSERR;
665 GNUNET_CONFIGURATION_get_value_string (config,
670 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
673 result = GNUNET_SYSERR;
678 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
680 scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL);
687 * @param argc argument count
688 * @param argv argument values
689 * @return 0 on success
692 main (int argc, char *const *argv)
694 static const struct GNUNET_GETOPT_CommandLineOption options[] = {
695 {'t', "table", "TABLENAME",
696 gettext_noop ("name of the table to write DFAs"),
697 1, &GNUNET_GETOPT_set_string, &table_name},
698 {'p', "max-path-compression", "MAX_PATH_COMPRESSION",
699 gettext_noop ("maximum path compression length"),
700 1, &GNUNET_GETOPT_set_uint, &max_path_compression},
701 GNUNET_GETOPT_OPTION_END
705 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
708 result = GNUNET_SYSERR;
710 GNUNET_PROGRAM_run (argc, argv,
711 "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
712 _("Profiler for regex library"), options, &run, NULL);
713 if (GNUNET_OK != ret)
715 if (GNUNET_OK != result)