From a4186fc2fd00b3fe2899bffcdbbbf8fead31115f Mon Sep 17 00:00:00 2001 From: lurchi Date: Mon, 25 Jun 2018 18:38:13 +0200 Subject: [PATCH] Revert "integrate dnsparser and dnsstub and tun with libgnunetutil" This reverts commit 7da98cf076e9c5101244dfbbf8c3ddff045d298e. --- configure.ac | 1 + po/POTFILES.in | 132 +- src/Makefile.am | 1 + src/dns/Makefile.am | 34 + src/{util => dns}/dnsparser.c | 0 src/{util => dns}/dnsstub.c | 0 src/{util => dns}/test_hexcoder.c | 0 src/exit/Makefile.am | 2 + src/gns/Makefile.am | 6 + src/gnsrecord/Makefile.am | 2 + src/include/gnunet_dnsstub_lib.h | 2 +- src/include/gnunet_util_lib.h | 2 - src/namestore/Makefile.am | 2 + src/pt/Makefile.am | 1 + src/regex/.gitignore | 12 + src/regex/Makefile.am | 213 + src/regex/gnunet-daemon-regexprofiler.c | 398 ++ src/regex/gnunet-regex-profiler.c | 1600 +++++++ src/regex/gnunet-regex-simulation-profiler.c | 723 ++++ src/regex/gnunet-service-regex.c | 416 ++ src/regex/perf-data.tar.gz | Bin 0 -> 22848 bytes src/regex/perf-regex.c | 126 + src/regex/plugin_block_regex.c | 402 ++ src/regex/regex.conf.in | 8 + src/regex/regex_api_announce.c | 184 + src/regex/regex_api_search.c | 248 ++ src/regex/regex_block_lib.c | 469 +++ src/regex/regex_block_lib.h | 191 + src/regex/regex_internal.c | 3706 +++++++++++++++++ src/regex/regex_internal.h | 453 ++ src/regex/regex_internal_dht.c | 829 ++++ src/regex/regex_internal_lib.h | 267 ++ src/regex/regex_ipc.h | 105 + src/regex/regex_simulation_profiler_test.conf | 7 + src/regex/regex_test_graph.c | 316 ++ src/regex/regex_test_lib.c | 646 +++ src/regex/regex_test_lib.h | 157 + src/regex/regex_test_random.c | 169 + src/regex/test_regex_api.c | 127 + src/regex/test_regex_api_data.conf | 39 + src/regex/test_regex_eval_api.c | 364 ++ src/regex/test_regex_graph_api.c | 156 + src/regex/test_regex_integration.c | 206 + src/regex/test_regex_iterate_api.c | 258 ++ src/regex/test_regex_proofs.c | 170 + src/tun/.gitignore | 2 + src/tun/Makefile.am | 46 + src/{util => tun}/regex.c | 0 src/{util => tun}/test_regex.c | 0 src/{util => tun}/test_tun.c | 0 src/{util => tun}/tun.c | 2 +- src/util/Makefile.am | 25 +- src/vpn/Makefile.am | 2 + 53 files changed, 13134 insertions(+), 93 deletions(-) rename src/{util => dns}/dnsparser.c (100%) rename src/{util => dns}/dnsstub.c (100%) rename src/{util => dns}/test_hexcoder.c (100%) create mode 100644 src/regex/.gitignore create mode 100644 src/regex/Makefile.am create mode 100644 src/regex/gnunet-daemon-regexprofiler.c create mode 100644 src/regex/gnunet-regex-profiler.c create mode 100644 src/regex/gnunet-regex-simulation-profiler.c create mode 100644 src/regex/gnunet-service-regex.c create mode 100644 src/regex/perf-data.tar.gz create mode 100644 src/regex/perf-regex.c create mode 100644 src/regex/plugin_block_regex.c create mode 100644 src/regex/regex.conf.in create mode 100644 src/regex/regex_api_announce.c create mode 100644 src/regex/regex_api_search.c create mode 100644 src/regex/regex_block_lib.c create mode 100644 src/regex/regex_block_lib.h create mode 100644 src/regex/regex_internal.c create mode 100644 src/regex/regex_internal.h create mode 100644 src/regex/regex_internal_dht.c create mode 100644 src/regex/regex_internal_lib.h create mode 100644 src/regex/regex_ipc.h create mode 100644 src/regex/regex_simulation_profiler_test.conf create mode 100644 src/regex/regex_test_graph.c create mode 100644 src/regex/regex_test_lib.c create mode 100644 src/regex/regex_test_lib.h create mode 100644 src/regex/regex_test_random.c create mode 100644 src/regex/test_regex_api.c create mode 100644 src/regex/test_regex_api_data.conf create mode 100644 src/regex/test_regex_eval_api.c create mode 100644 src/regex/test_regex_graph_api.c create mode 100644 src/regex/test_regex_integration.c create mode 100644 src/regex/test_regex_iterate_api.c create mode 100644 src/regex/test_regex_proofs.c create mode 100644 src/tun/.gitignore create mode 100644 src/tun/Makefile.am rename src/{util => tun}/regex.c (100%) rename src/{util => tun}/test_regex.c (100%) rename src/{util => tun}/test_tun.c (100%) rename src/{util => tun}/tun.c (99%) diff --git a/configure.ac b/configure.ac index 3d68ee9b0..c8e316416 100644 --- a/configure.ac +++ b/configure.ac @@ -1722,6 +1722,7 @@ src/testing/Makefile src/topology/Makefile src/transport/Makefile src/transport/transport.conf +src/tun/Makefile src/util/Makefile src/util/resolver.conf src/vpn/Makefile diff --git a/po/POTFILES.in b/po/POTFILES.in index 38fa52508..de6bd90e4 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -4,13 +4,21 @@ src/arm/arm_monitor_api.c src/arm/gnunet-arm.c src/arm/gnunet-service-arm.c src/arm/mockup-service.c +src/ats-tests/ats-testing-experiment.c +src/ats-tests/ats-testing-log.c +src/ats-tests/ats-testing-preferences.c +src/ats-tests/ats-testing-traffic.c +src/ats-tests/ats-testing.c +src/ats-tests/gnunet-ats-sim.c +src/ats-tests/gnunet-solver-eval.c +src/ats-tool/gnunet-ats.c src/ats/ats_api_connectivity.c src/ats/ats_api_performance.c src/ats/ats_api_scanner.c src/ats/ats_api_scheduling.c src/ats/gnunet-ats-solver-eval.c -src/ats/gnunet-service-ats_addresses.c src/ats/gnunet-service-ats.c +src/ats/gnunet-service-ats_addresses.c src/ats/gnunet-service-ats_connectivity.c src/ats/gnunet-service-ats_normalization.c src/ats/gnunet-service-ats_performance.c @@ -21,14 +29,6 @@ src/ats/gnunet-service-ats_scheduling.c src/ats/plugin_ats_mlp.c src/ats/plugin_ats_proportional.c src/ats/plugin_ats_ril.c -src/ats-tests/ats-testing.c -src/ats-tests/ats-testing-experiment.c -src/ats-tests/ats-testing-log.c -src/ats-tests/ats-testing-preferences.c -src/ats-tests/ats-testing-traffic.c -src/ats-tests/gnunet-ats-sim.c -src/ats-tests/gnunet-solver-eval.c -src/ats-tool/gnunet-ats.c src/auction/gnunet-auction-create.c src/auction/gnunet-auction-info.c src/auction/gnunet-auction-join.c @@ -40,8 +40,8 @@ src/block/plugin_block_test.c src/cadet/cadet_api.c src/cadet/cadet_test_lib.c src/cadet/desirability_table.c -src/cadet/gnunet-cadet.c src/cadet/gnunet-cadet-profiler.c +src/cadet/gnunet-cadet.c src/cadet/gnunet-service-cadet.c src/cadet/gnunet-service-cadet_channel.c src/cadet/gnunet-service-cadet_connection.c @@ -57,15 +57,15 @@ src/consensus/gnunet-service-consensus.c src/consensus/plugin_block_consensus.c src/conversation/conversation_api.c src/conversation/conversation_api_call.c -src/conversation/gnunet-conversation.c src/conversation/gnunet-conversation-test.c -src/conversation/gnunet_gst.c -src/conversation/gnunet_gst_test.c -src/conversation/gnunet-helper-audio-playback.c +src/conversation/gnunet-conversation.c src/conversation/gnunet-helper-audio-playback-gst.c -src/conversation/gnunet-helper-audio-record.c +src/conversation/gnunet-helper-audio-playback.c src/conversation/gnunet-helper-audio-record-gst.c +src/conversation/gnunet-helper-audio-record.c src/conversation/gnunet-service-conversation.c +src/conversation/gnunet_gst.c +src/conversation/gnunet_gst_test.c src/conversation/microphone.c src/conversation/plugin_gnsrecord_conversation.c src/conversation/speaker.c @@ -102,7 +102,6 @@ src/dht/dht_api.c src/dht/dht_test_lib.c src/dht/gnunet-dht-get.c src/dht/gnunet-dht-monitor.c -src/dht/gnunet_dht_profiler.c src/dht/gnunet-dht-put.c src/dht/gnunet-service-dht.c src/dht/gnunet-service-dht_clients.c @@ -111,8 +110,11 @@ src/dht/gnunet-service-dht_hello.c src/dht/gnunet-service-dht_neighbours.c src/dht/gnunet-service-dht_nse.c src/dht/gnunet-service-dht_routing.c +src/dht/gnunet_dht_profiler.c src/dht/plugin_block_dht.c src/dns/dns_api.c +src/dns/dnsparser.c +src/dns/dnsstub.c src/dns/gnunet-dns-monitor.c src/dns/gnunet-dns-redirector.c src/dns/gnunet-helper-dns.c @@ -124,8 +126,8 @@ src/dv/gnunet-dv.c src/dv/gnunet-service-dv.c src/dv/plugin_transport_dv.c src/exit/gnunet-daemon-exit.c -src/exit/gnunet-helper-exit.c src/exit/gnunet-helper-exit-windows.c +src/exit/gnunet-helper-exit.c src/fragmentation/defragmentation.c src/fragmentation/fragmentation.c src/fs/fs_api.c @@ -150,8 +152,8 @@ src/fs/gnunet-auto-share.c src/fs/gnunet-daemon-fsprofiler.c src/fs/gnunet-directory.c src/fs/gnunet-download.c -src/fs/gnunet-fs.c src/fs/gnunet-fs-profiler.c +src/fs/gnunet-fs.c src/fs/gnunet-helper-fs-publish.c src/fs/gnunet-publish.c src/fs/gnunet-search.c @@ -171,10 +173,10 @@ src/gns/gns_tld_api.c src/gns/gnunet-bcd.c src/gns/gnunet-dns2gns.c src/gns/gnunet-gns-benchmark.c -src/gns/gnunet-gns.c src/gns/gnunet-gns-helper-service-w32.c src/gns/gnunet-gns-import.c src/gns/gnunet-gns-proxy.c +src/gns/gnunet-gns.c src/gns/gnunet-service-gns.c src/gns/gnunet-service-gns_interceptor.c src/gns/gnunet-service-gns_resolver.c @@ -183,15 +185,15 @@ src/gns/nss/nss_gns_query.c src/gns/plugin_block_gns.c src/gns/plugin_gnsrecord_gns.c src/gns/plugin_rest_gns.c +src/gns/w32nsp-install.c +src/gns/w32nsp-resolve.c +src/gns/w32nsp-uninstall.c +src/gns/w32nsp.c src/gnsrecord/gnsrecord.c src/gnsrecord/gnsrecord_crypto.c src/gnsrecord/gnsrecord_misc.c src/gnsrecord/gnsrecord_serialization.c src/gnsrecord/plugin_gnsrecord_dns.c -src/gns/w32nsp.c -src/gns/w32nsp-install.c -src/gns/w32nsp-resolve.c -src/gns/w32nsp-uninstall.c src/hello/address.c src/hello/gnunet-hello.c src/hello/hello.c @@ -200,11 +202,6 @@ src/hostlist/gnunet-daemon-hostlist_client.c src/hostlist/gnunet-daemon-hostlist_server.c src/identity-attribute/identity_attribute.c src/identity-attribute/plugin_identity_attribute_gnuid.c -src/identity/gnunet-identity.c -src/identity/gnunet-service-identity.c -src/identity/identity_api.c -src/identity/identity_api_lookup.c -src/identity/plugin_rest_identity.c src/identity-provider/gnunet-idp.c src/identity-provider/gnunet-service-identity-provider.c src/identity-provider/identity_provider_api.c @@ -213,15 +210,20 @@ src/identity-provider/plugin_gnsrecord_identity_provider.c src/identity-provider/plugin_identity_provider_sqlite.c src/identity-provider/plugin_rest_identity_provider.c src/identity-provider/plugin_rest_openid_connect.c +src/identity/gnunet-identity.c +src/identity/gnunet-service-identity.c +src/identity/identity_api.c +src/identity/identity_api_lookup.c +src/identity/plugin_rest_identity.c +src/json/json.c +src/json/json_generator.c +src/json/json_helper.c +src/json/json_mhd.c src/jsonapi/jsonapi.c src/jsonapi/jsonapi_document.c src/jsonapi/jsonapi_error.c src/jsonapi/jsonapi_relationship.c src/jsonapi/jsonapi_resource.c -src/json/json.c -src/json/json_generator.c -src/json/json_helper.c -src/json/json_mhd.c src/multicast/gnunet-multicast.c src/multicast/gnunet-service-multicast.c src/multicast/multicast_api.c @@ -235,8 +237,8 @@ src/namecache/namecache_api.c src/namecache/plugin_namecache_flat.c src/namecache/plugin_namecache_postgres.c src/namecache/plugin_namecache_sqlite.c -src/namestore/gnunet-namestore.c src/namestore/gnunet-namestore-fcfsd.c +src/namestore/gnunet-namestore.c src/namestore/gnunet-service-namestore.c src/namestore/gnunet-zoneimport.c src/namestore/namestore_api.c @@ -252,10 +254,10 @@ src/nat-auto/gnunet-service-nat-auto.c src/nat-auto/gnunet-service-nat-auto_legacy.c src/nat-auto/nat_auto_api.c src/nat-auto/nat_auto_api_test.c -src/nat/gnunet-helper-nat-client.c src/nat/gnunet-helper-nat-client-windows.c -src/nat/gnunet-helper-nat-server.c +src/nat/gnunet-helper-nat-client.c src/nat/gnunet-helper-nat-server-windows.c +src/nat/gnunet-helper-nat-server.c src/nat/gnunet-nat.c src/nat/gnunet-service-nat.c src/nat/gnunet-service-nat_externalip.c @@ -264,15 +266,15 @@ src/nat/gnunet-service-nat_mini.c src/nat/gnunet-service-nat_stun.c src/nat/nat_api.c src/nat/nat_api_stun.c -src/nse/gnunet-nse.c src/nse/gnunet-nse-profiler.c +src/nse/gnunet-nse.c src/nse/gnunet-service-nse.c src/nse/nse_api.c +src/peerinfo-tool/gnunet-peerinfo.c +src/peerinfo-tool/gnunet-peerinfo_plugins.c src/peerinfo/gnunet-service-peerinfo.c src/peerinfo/peerinfo_api.c src/peerinfo/peerinfo_api_notify.c -src/peerinfo-tool/gnunet-peerinfo.c -src/peerinfo-tool/gnunet-peerinfo_plugins.c src/peerstore/gnunet-peerstore.c src/peerstore/gnunet-service-peerstore.c src/peerstore/peerstore_api.c @@ -317,20 +319,20 @@ src/revocation/gnunet-revocation.c src/revocation/gnunet-service-revocation.c src/revocation/plugin_block_revocation.c src/revocation/revocation_api.c -src/rps/gnunet-rps.c src/rps/gnunet-rps-profiler.c +src/rps/gnunet-rps.c src/rps/gnunet-service-rps.c src/rps/gnunet-service-rps_custommap.c src/rps/gnunet-service-rps_sampler.c src/rps/gnunet-service-rps_sampler_elem.c src/rps/gnunet-service-rps_view.c -src/rps/rps_api.c src/rps/rps-test_util.c +src/rps/rps_api.c src/scalarproduct/gnunet-scalarproduct.c -src/scalarproduct/gnunet-service-scalarproduct_alice.c -src/scalarproduct/gnunet-service-scalarproduct_bob.c src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c +src/scalarproduct/gnunet-service-scalarproduct_alice.c +src/scalarproduct/gnunet-service-scalarproduct_bob.c src/scalarproduct/scalarproduct_api.c src/secretsharing/gnunet-secretsharing-profiler.c src/secretsharing/gnunet-service-secretsharing.c @@ -359,15 +361,16 @@ src/statistics/gnunet-statistics.c src/statistics/statistics_api.c src/template/gnunet-service-template.c src/template/gnunet-template.c +src/testbed-logger/gnunet-service-testbed-logger.c +src/testbed-logger/testbed_logger_api.c src/testbed/generate-underlay-topology.c src/testbed/gnunet-daemon-latency-logger.c src/testbed/gnunet-daemon-testbed-blacklist.c src/testbed/gnunet-daemon-testbed-underlay.c src/testbed/gnunet-helper-testbed.c -src/testbed/gnunet_mpi_test.c src/testbed/gnunet-service-test-barriers.c -src/testbed/gnunet-service-testbed_barriers.c src/testbed/gnunet-service-testbed.c +src/testbed/gnunet-service-testbed_barriers.c src/testbed/gnunet-service-testbed_cache.c src/testbed/gnunet-service-testbed_connectionpool.c src/testbed/gnunet-service-testbed_cpustatus.c @@ -375,20 +378,19 @@ src/testbed/gnunet-service-testbed_links.c src/testbed/gnunet-service-testbed_meminfo.c src/testbed/gnunet-service-testbed_oc.c src/testbed/gnunet-service-testbed_peers.c -src/testbed/gnunet_testbed_mpi_spawn.c src/testbed/gnunet-testbed-profiler.c -src/testbed-logger/gnunet-service-testbed-logger.c -src/testbed-logger/testbed_logger_api.c -src/testbed/testbed_api_barriers.c +src/testbed/gnunet_mpi_test.c +src/testbed/gnunet_testbed_mpi_spawn.c src/testbed/testbed_api.c +src/testbed/testbed_api_barriers.c src/testbed/testbed_api_hosts.c src/testbed/testbed_api_operations.c src/testbed/testbed_api_peers.c src/testbed/testbed_api_sd.c src/testbed/testbed_api_services.c src/testbed/testbed_api_statistics.c -src/testbed/testbed_api_testbed.c src/testbed/testbed_api_test.c +src/testbed/testbed_api_testbed.c src/testbed/testbed_api_topology.c src/testbed/testbed_api_underlay.c src/testing/gnunet-testing.c @@ -397,28 +399,28 @@ src/testing/testing.c src/topology/friends.c src/topology/gnunet-daemon-topology.c src/transport/gnunet-helper-transport-bluetooth.c -src/transport/gnunet-helper-transport-wlan.c src/transport/gnunet-helper-transport-wlan-dummy.c -src/transport/gnunet-service-transport_ats.c +src/transport/gnunet-helper-transport-wlan.c src/transport/gnunet-service-transport.c +src/transport/gnunet-service-transport_ats.c src/transport/gnunet-service-transport_hello.c src/transport/gnunet-service-transport_manipulation.c src/transport/gnunet-service-transport_neighbours.c src/transport/gnunet-service-transport_plugins.c src/transport/gnunet-service-transport_validation.c -src/transport/gnunet-transport.c src/transport/gnunet-transport-certificate-creation.c src/transport/gnunet-transport-profiler.c src/transport/gnunet-transport-wlan-receiver.c src/transport/gnunet-transport-wlan-sender.c +src/transport/gnunet-transport.c src/transport/plugin_transport_http_client.c src/transport/plugin_transport_http_common.c src/transport/plugin_transport_http_server.c src/transport/plugin_transport_smtp.c src/transport/plugin_transport_tcp.c src/transport/plugin_transport_template.c -src/transport/plugin_transport_udp_broadcasting.c src/transport/plugin_transport_udp.c +src/transport/plugin_transport_udp_broadcasting.c src/transport/plugin_transport_unix.c src/transport/plugin_transport_wlan.c src/transport/plugin_transport_xt.c @@ -427,6 +429,11 @@ src/transport/tcp_connection_legacy.c src/transport/tcp_server_legacy.c src/transport/tcp_server_mst_legacy.c src/transport/tcp_service_legacy.c +src/transport/transport-testing-filenames.c +src/transport/transport-testing-loggers.c +src/transport/transport-testing-main.c +src/transport/transport-testing-send.c +src/transport/transport-testing.c src/transport/transport_api_address_to_string.c src/transport/transport_api_blacklist.c src/transport/transport_api_core.c @@ -435,11 +442,8 @@ src/transport/transport_api_manipulation.c src/transport/transport_api_monitor_peers.c src/transport/transport_api_monitor_plugins.c src/transport/transport_api_offer_hello.c -src/transport/transport-testing.c -src/transport/transport-testing-filenames.c -src/transport/transport-testing-loggers.c -src/transport/transport-testing-main.c -src/transport/transport-testing-send.c +src/tun/regex.c +src/tun/tun.c src/util/bandwidth.c src/util/bio.c src/util/client.c @@ -451,8 +455,8 @@ src/util/configuration_loader.c src/util/container_bloomfilter.c src/util/container_heap.c src/util/container_meta_data.c -src/util/container_multihashmap32.c src/util/container_multihashmap.c +src/util/container_multihashmap32.c src/util/container_multipeermap.c src/util/container_multishortmap.c src/util/crypto_abe.c @@ -470,12 +474,10 @@ src/util/crypto_random.c src/util/crypto_rsa.c src/util/crypto_symmetric.c src/util/disk.c -src/util/dnsparser.c -src/util/dnsstub.c src/util/getopt.c src/util/getopt_helpers.c -src/util/gnunet-config.c src/util/gnunet-config-diff.c +src/util/gnunet-config.c src/util/gnunet-ecc.c src/util/gnunet-helper-w32-console.c src/util/gnunet-resolver.c @@ -495,7 +497,6 @@ src/util/os_priority.c src/util/peer.c src/util/plugin.c src/util/program.c -src/util/regex.c src/util/resolver_api.c src/util/scheduler.c src/util/service.c @@ -504,17 +505,16 @@ src/util/socks.c src/util/speedup.c src/util/strings.c src/util/time.c -src/util/tun.c src/util/w32cat.c src/util/win.c src/util/winproc.c -src/vpn/gnunet-helper-vpn.c src/vpn/gnunet-helper-vpn-windows.c +src/vpn/gnunet-helper-vpn.c src/vpn/gnunet-service-vpn.c src/vpn/gnunet-vpn.c src/vpn/vpn_api.c -src/zonemaster/gnunet-service-zonemaster.c src/zonemaster/gnunet-service-zonemaster-monitor.c +src/zonemaster/gnunet-service-zonemaster.c src/fs/fs_api.h src/include/gnunet_common.h src/include/gnunet_mq_lib.h diff --git a/src/Makefile.am b/src/Makefile.am index 00f30adc3..d8d548706 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -83,6 +83,7 @@ SUBDIRS = \ $(REST_DIR) \ $(JSONAPI_DIR) \ hello \ + tun \ block \ statistics \ arm \ diff --git a/src/dns/Makefile.am b/src/dns/Makefile.am index ca2685765..9a4ecdcfd 100644 --- a/src/dns/Makefile.am +++ b/src/dns/Makefile.am @@ -27,6 +27,8 @@ install-exec-hook: endif lib_LTLIBRARIES = \ + libgnunetdnsparser.la \ + libgnunetdnsstub.la \ libgnunetdns.la libexec_PROGRAMS = \ @@ -45,6 +47,9 @@ check_SCRIPTS = \ test_gnunet_dns.sh endif +check_PROGRAMS = \ + test_hexcoder + gnunet_helper_dns_SOURCES = \ gnunet-helper-dns.c @@ -52,6 +57,7 @@ gnunet_helper_dns_SOURCES = \ gnunet_dns_monitor_SOURCES = \ gnunet-dns-monitor.c gnunet_dns_monitor_LDADD = \ + libgnunetdnsparser.la \ libgnunetdns.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) @@ -59,12 +65,15 @@ gnunet_dns_monitor_LDADD = \ gnunet_zonewalk_SOURCES = \ gnunet-zonewalk.c gnunet_zonewalk_LDADD = \ + libgnunetdnsparser.la \ + libgnunetdnsstub.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) gnunet_dns_redirector_SOURCES = \ gnunet-dns-redirector.c gnunet_dns_redirector_LDADD = \ + libgnunetdnsparser.la \ libgnunetdns.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) @@ -72,10 +81,30 @@ gnunet_dns_redirector_LDADD = \ gnunet_service_dns_SOURCES = \ gnunet-service-dns.c gnunet_service_dns_LDADD = \ + libgnunetdnsstub.la \ + $(top_builddir)/src/tun/libgnunettun.la \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) +libgnunetdnsparser_la_SOURCES = \ + dnsparser.c +libgnunetdnsparser_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIB) \ + -lidn +libgnunetdnsparser_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) \ + -version-info 1:0:1 + +libgnunetdnsstub_la_SOURCES = \ + dnsstub.c +libgnunetdnsstub_la_LIBADD = \ + $(top_builddir)/src/tun/libgnunettun.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIB) +libgnunetdnsstub_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) \ + -version-info 0:0:0 + libgnunetdns_la_SOURCES = \ dns_api.c dns.h libgnunetdns_la_LIBADD = \ @@ -102,3 +131,8 @@ EXTRA_DIST = \ $(check_SCRIPTS) +test_hexcoder_SOURCES = \ + test_hexcoder.c +test_hexcoder_LDADD = \ + libgnunetdnsparser.la \ + $(top_builddir)/src/util/libgnunetutil.la diff --git a/src/util/dnsparser.c b/src/dns/dnsparser.c similarity index 100% rename from src/util/dnsparser.c rename to src/dns/dnsparser.c diff --git a/src/util/dnsstub.c b/src/dns/dnsstub.c similarity index 100% rename from src/util/dnsstub.c rename to src/dns/dnsstub.c diff --git a/src/util/test_hexcoder.c b/src/dns/test_hexcoder.c similarity index 100% rename from src/util/test_hexcoder.c rename to src/dns/test_hexcoder.c diff --git a/src/exit/Makefile.am b/src/exit/Makefile.am index ea4f08c73..aa1210269 100644 --- a/src/exit/Makefile.am +++ b/src/exit/Makefile.am @@ -49,8 +49,10 @@ endif gnunet_daemon_exit_SOURCES = \ gnunet-daemon-exit.c exit.h gnunet_daemon_exit_LDADD = \ + $(top_builddir)/src/dns/libgnunetdnsstub.la \ $(top_builddir)/src/dht/libgnunetdht.la \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/tun/libgnunettun.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(top_builddir)/src/cadet/libgnunetcadet.la \ $(top_builddir)/src/regex/libgnunetregex.la \ diff --git a/src/gns/Makefile.am b/src/gns/Makefile.am index 2c7bb8ebb..46642113f 100644 --- a/src/gns/Makefile.am +++ b/src/gns/Makefile.am @@ -102,6 +102,7 @@ libgnunet_plugin_gnsrecord_gns_la_SOURCES = \ plugin_gnsrecord_gns.c libgnunet_plugin_gnsrecord_gns_la_LIBADD = \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(LTLIBINTL) libgnunet_plugin_gnsrecord_gns_la_LDFLAGS = \ @@ -139,6 +140,8 @@ gnunet_dns2gns_LDADD = \ libgnunetgns.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(top_builddir)/src/identity/libgnunetidentity.la \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ + $(top_builddir)/src/dns/libgnunetdnsstub.la \ $(GN_LIBINTL) if LINUX @@ -203,7 +206,10 @@ gnunet_service_gns_LDADD = \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(top_builddir)/src/dns/libgnunetdns.la \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ + $(top_builddir)/src/dns/libgnunetdnsstub.la \ $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/tun/libgnunettun.la \ $(top_builddir)/src/namecache/libgnunetnamecache.la \ $(USE_VPN) \ $(GN_LIBINTL) diff --git a/src/gnsrecord/Makefile.am b/src/gnsrecord/Makefile.am index f840a31a4..c83aa3307 100644 --- a/src/gnsrecord/Makefile.am +++ b/src/gnsrecord/Makefile.am @@ -38,6 +38,7 @@ libgnunetgnsrecord_la_SOURCES = \ gnsrecord_crypto.c \ gnsrecord_misc.c libgnunetgnsrecord_la_LIBADD = \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) libgnunetgnsrecord_la_LDFLAGS = \ @@ -52,6 +53,7 @@ plugin_LTLIBRARIES = \ libgnunet_plugin_gnsrecord_dns_la_SOURCES = \ plugin_gnsrecord_dns.c libgnunet_plugin_gnsrecord_dns_la_LIBADD = \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(LTLIBINTL) libgnunet_plugin_gnsrecord_dns_la_LDFLAGS = \ diff --git a/src/include/gnunet_dnsstub_lib.h b/src/include/gnunet_dnsstub_lib.h index 9b10c5c48..8f1b90425 100644 --- a/src/include/gnunet_dnsstub_lib.h +++ b/src/include/gnunet_dnsstub_lib.h @@ -85,7 +85,7 @@ GNUNET_DNSSTUB_add_dns_sa (struct GNUNET_DNSSTUB_Context *ctx, * Only effective for requests issued after this call. * * @param ctx resolver context to modify - * @param retry_freq how long to wait between retries + * @param retry_frequ how long to wait between retries */ void GNUNET_DNSSTUB_set_retry (struct GNUNET_DNSSTUB_Context *ctx, diff --git a/src/include/gnunet_util_lib.h b/src/include/gnunet_util_lib.h index 82f61a22b..5e8eef1ad 100644 --- a/src/include/gnunet_util_lib.h +++ b/src/include/gnunet_util_lib.h @@ -78,8 +78,6 @@ extern "C" #include "gnunet_service_lib.h" #include "gnunet_signal_lib.h" #include "gnunet_strings_lib.h" -#include "gnunet_dnsparser_lib.h" -#include "gnunet_dnsstub_lib.h" #if 0 /* keep Emacsens' auto-indent happy */ { diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am index 777e722c2..fa85cc060 100644 --- a/src/namestore/Makefile.am +++ b/src/namestore/Makefile.am @@ -147,6 +147,8 @@ gnunet_zoneimport_LDADD = \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ $(top_builddir)/src/identity/libgnunetidentity.la \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ + $(top_builddir)/src/dns/libgnunetdnsstub.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) diff --git a/src/pt/Makefile.am b/src/pt/Makefile.am index 188387c0c..e36630ae4 100644 --- a/src/pt/Makefile.am +++ b/src/pt/Makefile.am @@ -28,6 +28,7 @@ gnunet_daemon_pt_LDADD = \ $(top_builddir)/src/cadet/libgnunetcadet.la \ $(top_builddir)/src/dht/libgnunetdht.la \ $(top_builddir)/src/dns/libgnunetdns.la \ + $(top_builddir)/src/dns/libgnunetdnsparser.la \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) diff --git a/src/regex/.gitignore b/src/regex/.gitignore new file mode 100644 index 000000000..39dc89c88 --- /dev/null +++ b/src/regex/.gitignore @@ -0,0 +1,12 @@ +perf-regex +gnunet-daemon-regexprofiler +gnunet-regex-profiler +gnunet-regex-simulation-profiler +gnunet-service-regex +test_graph.dot +test_regex_api +test_regex_eval_api +test_regex_graph_api +test_regex_integration +test_regex_iterate_api +test_regex_proofs diff --git a/src/regex/Makefile.am b/src/regex/Makefile.am new file mode 100644 index 000000000..80997db40 --- /dev/null +++ b/src/regex/Makefile.am @@ -0,0 +1,213 @@ +# This Makefile.am is in the public domain +AM_CPPFLAGS = -I$(top_srcdir)/src/include + +if MINGW + WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +endif + +if USE_COVERAGE + AM_CFLAGS = --coverage +endif + +pkgcfgdir= $(pkgdatadir)/config.d/ + +libexecdir= $(pkglibdir)/libexec/ + +plugindir = $(libdir)/gnunet + +pkgcfg_DATA = \ + regex.conf + +libexec_PROGRAMS = \ + gnunet-service-regex \ + gnunet-daemon-regexprofiler + + +gnunet_service_regex_SOURCES = \ + gnunet-service-regex.c +gnunet_service_regex_LDADD = -lm \ + libgnunetregex_internal.a \ + libgnunetregexblock.la \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +noinst_LIBRARIES = \ + libgnunetregex_internal.a \ + libgnunetregextest.a + +lib_LTLIBRARIES = \ + libgnunetregexblock.la \ + libgnunetregex.la + + +libgnunetregexblock_la_SOURCES = \ + regex_block_lib.c regex_block_lib.h +libgnunetregexblock_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(XLIB) \ + $(LTLIBINTL) +libgnunetregexblock_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 1:0:0 + + +libgnunetregex_internal_a_SOURCES = \ + regex_internal_lib.h \ + regex_internal.h regex_internal.c \ + regex_internal_dht.c +libgnunetregex_internal_a_DEPENDENCIES = \ + libgnunetregexblock.la + + +libgnunetregex_la_SOURCES = \ + regex_api_announce.c \ + regex_api_search.c \ + regex_ipc.h +libgnunetregex_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la +libgnunetregex_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) \ + -version-info 3:1:0 + + +plugin_LTLIBRARIES = \ + libgnunet_plugin_block_regex.la + +libgnunet_plugin_block_regex_la_SOURCES = \ + plugin_block_regex.c +libgnunet_plugin_block_regex_la_LIBADD = \ + libgnunetregexblock.la \ + $(top_builddir)/src/block/libgnunetblock.la \ + $(top_builddir)/src/block/libgnunetblockgroup.la \ + $(top_builddir)/src/util/libgnunetutil.la +libgnunet_plugin_block_regex_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + +if HAVE_MYSQL +noinst_mysql_progs = \ + gnunet-regex-simulation-profiler + +gnunet_regex_simulation_profiler_SOURCES = \ + gnunet-regex-simulation-profiler.c +gnunet_regex_simulation_profiler_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/my/libgnunetmy.la \ + $(top_builddir)/src/mysql/libgnunetmysql.la +endif + +libgnunetregextest_a_SOURCES = \ + regex_test_lib.c regex_test_lib.h \ + regex_test_graph.c \ + regex_test_random.c +libgnunetregextest_a_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + libgnunetregex_internal.a + +if HAVE_TESTING +noinst_PROGRAMS = $(noinst_mysql_progs) \ + perf-regex \ + gnunet-regex-profiler +endif + +perf_regex_SOURCES = \ + perf-regex.c +perf_regex_LDADD = -lm \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregexblock.la \ + libgnunetregextest.a \ + $(top_builddir)/src/util/libgnunetutil.la + +gnunet_regex_profiler_SOURCES = \ + gnunet-regex-profiler.c +gnunet_regex_profiler_LDADD = -lm \ + $(top_builddir)/src/arm/libgnunetarm.la \ + $(top_builddir)/src/testbed/libgnunettestbed.la \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregexblock.la \ + libgnunetregextest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la + +gnunet_daemon_regexprofiler_SOURCES = \ + gnunet-daemon-regexprofiler.c +gnunet_daemon_regexprofiler_LDADD = -lm \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregexblock.la \ + libgnunetregextest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la + +check_PROGRAMS = \ + test_regex_integration \ + test_regex_eval_api \ + test_regex_iterate_api \ + test_regex_proofs \ + test_regex_graph_api \ + test_regex_api + +if ENABLE_TEST_RUN + AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME; + TESTS = $(check_PROGRAMS) +endif + +test_regex_eval_api_SOURCES = \ + test_regex_eval_api.c +test_regex_eval_api_LDADD = -lm \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregextest.a \ + libgnunetregexblock.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_regex_integration_SOURCES = \ + test_regex_integration.c +test_regex_integration_LDADD = -lm \ + libgnunetregex.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/tun/libgnunettun.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_regex_api_SOURCES = \ + test_regex_api.c +test_regex_api_LDADD = -lm \ + libgnunetregex.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_regex_iterate_api_SOURCES = \ + test_regex_iterate_api.c +test_regex_iterate_api_LDADD = -lm \ + libgnunetregex_internal.a \ + libgnunetregexblock.la \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_regex_proofs_SOURCES = \ + test_regex_proofs.c +test_regex_proofs_LDADD = -lm \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregextest.a \ + libgnunetregexblock.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_regex_graph_api_SOURCES = \ + test_regex_graph_api.c +test_regex_graph_api_LDADD = -lm \ + libgnunetregex_internal.a \ + $(top_builddir)/src/dht/libgnunetdht.la \ + libgnunetregextest.a \ + libgnunetregexblock.la \ + $(top_builddir)/src/util/libgnunetutil.la + + +EXTRA_DIST = \ + regex_simulation_profiler_test.conf \ + test_regex_api_data.conf diff --git a/src/regex/gnunet-daemon-regexprofiler.c b/src/regex/gnunet-daemon-regexprofiler.c new file mode 100644 index 000000000..11c2f513a --- /dev/null +++ b/src/regex/gnunet-daemon-regexprofiler.c @@ -0,0 +1,398 @@ +/* + This file is part of GNUnet. + Copyright (C) 2012, 2013 Christian Grothoff + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +/** + * @file regex/gnunet-daemon-regexprofiler.c + * @brief daemon that uses cadet to announce a regular expression. Used in + * conjunction with gnunet-regex-profiler to announce regexes on serveral peers + * without the need to explicitly connect to the cadet service running on the + * peer from within the profiler. + * @author Maximilian Szengel + * @author Bartlomiej Polot + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" +#include "gnunet_dht_service.h" +#include "gnunet_statistics_service.h" + +/** + * Return value from 'main'. + */ +static int global_ret; + +/** + * Configuration we use. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Handle to the statistics service. + */ +static struct GNUNET_STATISTICS_Handle *stats_handle; + +/** + * Peer's dht handle. + */ +static struct GNUNET_DHT_Handle *dht_handle; + +/** + * Peer's regex announce handle. + */ +static struct REGEX_INTERNAL_Announcement *announce_handle; + +/** + * Periodically reannounce regex. + */ +static struct GNUNET_SCHEDULER_Task * reannounce_task; + +/** + * What's the maximum reannounce period. + */ +static struct GNUNET_TIME_Relative reannounce_period_max; + +/** + * Maximal path compression length for regex announcing. + */ +static unsigned long long max_path_compression; + +/** + * Name of the file containing policies that this peer should announce. One + * policy per line. + */ +static char * policy_filename; + +/** + * Prefix to add before every regex we're announcing. + */ +static char * regex_prefix; + +/** + * Regex with prefix. + */ +static char *rx_with_pfx; + +/** + * How many put rounds should we do. + */ +static unsigned int rounds = 3; + +/** + * Private key for this peer. + */ +static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key; + + + +/** + * Task run during shutdown. + * + * @param cls unused + */ +static void +shutdown_task (void *cls) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "shutting down\n"); + + if (NULL != announce_handle) + { + REGEX_INTERNAL_announce_cancel (announce_handle); + announce_handle = NULL; + } + if (NULL != reannounce_task) + { + GNUNET_free (GNUNET_SCHEDULER_cancel (reannounce_task)); + reannounce_task = NULL; + } + if (NULL != dht_handle) + { + GNUNET_DHT_disconnect (dht_handle); + dht_handle = NULL; + } + GNUNET_free (my_private_key); + my_private_key = NULL; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Daemon for %s shutting down\n", + policy_filename); +} + + +/** + * Announce a previously announced regex re-using cached data. + * + * @param cls Closure (regex to announce if needed). + */ +static void +reannounce_regex (void *cls) +{ + char *regex = cls; + struct GNUNET_TIME_Relative random_delay; + + reannounce_task = NULL; + if (0 == rounds--) + { + global_ret = 0; + GNUNET_SCHEDULER_shutdown (); + GNUNET_free (regex); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Announcing regex: %s\n", regex); + GNUNET_STATISTICS_update (stats_handle, "# regexes announced", 1, GNUNET_NO); + if (NULL == announce_handle && NULL != regex) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "First time, creating regex: %s\n", + regex); + announce_handle = REGEX_INTERNAL_announce (dht_handle, + my_private_key, + regex, + (unsigned int) max_path_compression, + stats_handle); + } + else + { + GNUNET_assert (NULL != announce_handle); + REGEX_INTERNAL_reannounce (announce_handle); + } + + random_delay = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, + GNUNET_CRYPTO_random_u32 ( + GNUNET_CRYPTO_QUALITY_WEAK, + reannounce_period_max.rel_value_us)); + reannounce_task = GNUNET_SCHEDULER_add_delayed (random_delay, + &reannounce_regex, cls); +} + + +/** + * Announce the given regular expression using regex and the path compression + * length read from config. + * + * @param regex regular expression to announce on this peer's cadet. + */ +static void +announce_regex (const char *regex) +{ + char *copy; + + if (NULL == regex || 0 == strlen (regex)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Cannot announce empty regex\n"); + return; + } + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Daemon for %s starting\n", + policy_filename); + GNUNET_assert (NULL == reannounce_task); + copy = GNUNET_strdup (regex); + reannounce_task = GNUNET_SCHEDULER_add_now (&reannounce_regex, + (void *) copy); +} + + +/** + * Scan through the policy_dir looking for the n-th filename. + * + * @param cls Closure (target number n). + * @param filename complete filename (absolute path). + * @return GNUNET_OK to continue to iterate, + * GNUNET_NO to stop when found + */ +static int +scan (void *cls, const char *filename) +{ + long n = (long) cls; + static long c = 0; + + if (c == n) + { + policy_filename = GNUNET_strdup (filename); + return GNUNET_NO; + } + c++; + return GNUNET_OK; +} + + +/** + * @brief Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param cfg_ configuration + */ +static void +run (void *cls, char *const *args GNUNET_UNUSED, + const char *cfgfile GNUNET_UNUSED, + const struct GNUNET_CONFIGURATION_Handle *cfg_) +{ + char *regex = NULL; + char **components; + char *policy_dir; + long long unsigned int peer_id; + + cfg = cfg_; + + my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (cfg); + GNUNET_assert (NULL != my_private_key); + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER", + "MAX_PATH_COMPRESSION", + &max_path_compression)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _ + ("%s service is lacking key configuration settings (%s). Exiting.\n"), + "regexprofiler", "max_path_compression"); + global_ret = GNUNET_SYSERR; + GNUNET_SCHEDULER_shutdown (); + return; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER", + "POLICY_DIR", &policy_dir)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "REGEXPROFILER", "POLICY_DIR"); + global_ret = GNUNET_SYSERR; + GNUNET_SCHEDULER_shutdown (); + return; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "TESTBED", + "PEERID", &peer_id)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "TESTBED", "PEERID"); + global_ret = GNUNET_SYSERR; + GNUNET_free (policy_dir); + GNUNET_SCHEDULER_shutdown (); + return; + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER", + "REGEX_PREFIX", ®ex_prefix)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "REGEXPROFILER", "REGEX_PREFIX"); + global_ret = GNUNET_SYSERR; + GNUNET_free (policy_dir); + GNUNET_SCHEDULER_shutdown (); + return; + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER", + "REANNOUNCE_PERIOD_MAX", + &reannounce_period_max)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "reannounce_period_max not given. Using 10 minutes.\n"); + reannounce_period_max = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10); + } + + stats_handle = GNUNET_STATISTICS_create ("regexprofiler", cfg); + + dht_handle = GNUNET_DHT_connect (cfg, 1); + + if (NULL == dht_handle) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not acquire dht handle. Exiting.\n"); + global_ret = GNUNET_SYSERR; + GNUNET_free (policy_dir); + GNUNET_SCHEDULER_shutdown (); + return; + } + + /* Read regexes from policy files */ + GNUNET_assert (-1 != GNUNET_DISK_directory_scan (policy_dir, &scan, + (void *) (long) peer_id)); + if (NULL == (components = REGEX_TEST_read_from_file (policy_filename))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Policy file %s contains no policies. Exiting.\n", + policy_filename); + global_ret = GNUNET_SYSERR; + GNUNET_free (policy_dir); + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_free (policy_dir); + regex = REGEX_TEST_combine (components, 16); + REGEX_TEST_free_from_file (components); + + /* Announcing regexes from policy_filename */ + GNUNET_asprintf (&rx_with_pfx, + "%s(%s)(0|1|2|3|4|5|6|7|8|9|a|b|c|d|e|f)*", + regex_prefix, + regex); + announce_regex (rx_with_pfx); + GNUNET_free (regex); + GNUNET_free (rx_with_pfx); + + /* Scheduled the task to clean up when shutdown is called */ + GNUNET_SCHEDULER_add_shutdown (&shutdown_task, + NULL); +} + + +/** + * The main function of the regexprofiler service. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) + return 2; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "regexprofiler", + gettext_noop + ("Daemon to announce regular expressions for the peer using cadet."), + options, &run, NULL)) ? global_ret : 1; +} + + +#if defined(LINUX) && defined(__GLIBC__) +#include + +/** + * MINIMIZE heap size (way below 128k) since this process doesn't need much. + */ +void __attribute__ ((constructor)) GNUNET_ARM_memory_init () +{ + mallopt (M_TRIM_THRESHOLD, 4 * 1024); + mallopt (M_TOP_PAD, 1 * 1024); + malloc_trim (0); +} +#endif + + +/* end of gnunet-daemon-regexprofiler.c */ diff --git a/src/regex/gnunet-regex-profiler.c b/src/regex/gnunet-regex-profiler.c new file mode 100644 index 000000000..f65ba3db1 --- /dev/null +++ b/src/regex/gnunet-regex-profiler.c @@ -0,0 +1,1600 @@ +/* + This file is part of GNUnet. + Copyright (C) 2011 - 2017 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +/** + * @file regex/gnunet-regex-profiler.c + * @brief Regex profiler for testing distributed regex use. + * @author Bartlomiej Polot + * @author Maximilian Szengel + * + */ + +#include + +#include "platform.h" +#include "gnunet_applications.h" +#include "gnunet_util_lib.h" +#include "regex_internal_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_testbed_service.h" + +#define FIND_TIMEOUT \ + GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90) + +/** + * DLL of operations + */ +struct DLLOperation +{ + /** + * The testbed operation handle + */ + struct GNUNET_TESTBED_Operation *op; + + /** + * Closure + */ + void *cls; + + /** + * The next pointer for DLL + */ + struct DLLOperation *next; + + /** + * The prev pointer for DLL + */ + struct DLLOperation *prev; +}; + + +/** + * Available states during profiling + */ +enum State +{ + /** + * Initial state + */ + STATE_INIT = 0, + + /** + * Starting slaves + */ + STATE_SLAVES_STARTING, + + /** + * Creating peers + */ + STATE_PEERS_CREATING, + + /** + * Starting peers + */ + STATE_PEERS_STARTING, + + /** + * Linking peers + */ + STATE_PEERS_LINKING, + + /** + * Matching strings against announced regexes + */ + STATE_SEARCH_REGEX, + + /** + * Destroying peers; we can do this as the controller takes care of stopping a + * peer if it is running + */ + STATE_PEERS_DESTROYING +}; + + +/** + * Peer handles. + */ +struct RegexPeer +{ + /** + * Peer id. + */ + unsigned int id; + + /** + * Peer configuration handle. + */ + struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * The actual testbed peer handle. + */ + struct GNUNET_TESTBED_Peer *peer_handle; + + /** + * Peer's search string. + */ + const char *search_str; + + /** + * Set to GNUNET_YES if the peer successfully matched the above + * search string. GNUNET_NO if the string could not be matched + * during the profiler run. GNUNET_SYSERR if the string matching + * timed out. Undefined if search_str is NULL + */ + int search_str_matched; + + /** + * Peer's DHT handle. + */ + struct GNUNET_DHT_Handle *dht_handle; + + /** + * Handle to a running regex search. + */ + struct REGEX_INTERNAL_Search *search_handle; + + /** + * Testbed operation handle for DHT. + */ + struct GNUNET_TESTBED_Operation *op_handle; + + /** + * Peers's statistics handle. + */ + struct GNUNET_STATISTICS_Handle *stats_handle; + + /** + * The starting time of a profiling step. + */ + struct GNUNET_TIME_Absolute prof_start_time; + + /** + * Operation timeout + */ + struct GNUNET_SCHEDULER_Task * timeout; + + /** + * Deamon start + */ + struct GNUNET_TESTBED_Operation *daemon_op; +}; + +/** + * Set when shutting down to avoid making more queries. + */ +static int in_shutdown; + +/** + * The array of peers; we fill this as the peers are given to us by the testbed + */ +static struct RegexPeer *peers; + +/** + * Host registration handle + */ +static struct GNUNET_TESTBED_HostRegistrationHandle *reg_handle; + +/** + * Handle to the master controller process + */ +static struct GNUNET_TESTBED_ControllerProc *mc_proc; + +/** + * Handle to the master controller + */ +static struct GNUNET_TESTBED_Controller *mc; + +/** + * Handle to global configuration + */ +static struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Abort task identifier + */ +static struct GNUNET_SCHEDULER_Task * abort_task; + +/** + * Host registration task identifier + */ +static struct GNUNET_SCHEDULER_Task * register_hosts_task; + +/** + * Global event mask for all testbed events + */ +static uint64_t event_mask; + +/** + * The starting time of a profiling step + */ +static struct GNUNET_TIME_Absolute prof_start_time; + +/** + * Duration profiling step has taken + */ +static struct GNUNET_TIME_Relative prof_time; + +/** + * Number of peers to be started by the profiler + */ +static unsigned int num_peers; + +/** + * Global testing status + */ +static int result; + +/** + * current state of profiling + */ +enum State state; + +/** + * Folder where policy files are stored. + */ +static char * policy_dir; + +/** + * File with hostnames where to execute the test. + */ +static char *hosts_file; + +/** + * File with the strings to look for. + */ +static char *strings_file; + +/** + * Search strings (num_peers of them). + */ +static char **search_strings; + +/** + * How many searches are we going to start in parallel + */ +static long long unsigned int init_parallel_searches; + +/** + * How many searches are running in parallel + */ +static unsigned int parallel_searches; + +/** + * Number of strings found in the published regexes. + */ +static unsigned int strings_found; + +/** + * Index of peer to start next announce/search. + */ +static unsigned int next_search; + +/** + * Search timeout task identifier. + */ +static struct GNUNET_SCHEDULER_Task * search_timeout_task; + +/** + * Search timeout in seconds. + */ +static struct GNUNET_TIME_Relative search_timeout_time = { 60000 }; + +/** + * File to log statistics to. + */ +static struct GNUNET_DISK_FileHandle *data_file; + +/** + * Filename to log statistics to. + */ +static char *data_filename; + +/** + * Prefix used for regex announcing. We need to prefix the search + * strings with it, in order to find something. + */ +static char * regex_prefix; + +/** + * What's the maximum regex reannounce period. + */ +static struct GNUNET_TIME_Relative reannounce_period_max; + + +/******************************************************************************/ +/****************************** DECLARATIONS ********************************/ +/******************************************************************************/ + +/** + * DHT connect callback. + * + * @param cls internal peer id. + * @param op operation handle. + * @param ca_result connect adapter result. + * @param emsg error message. + */ +static void +dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op, + void *ca_result, const char *emsg); + +/** + * DHT connect adapter. + * + * @param cls not used. + * @param cfg configuration handle. + * + * @return + */ +static void * +dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg); + + +/** + * Adapter function called to destroy a connection to + * the DHT service + * + * @param cls closure + * @param op_result service handle returned from the connect adapter + */ +static void +dht_da (void *cls, void *op_result); + + +/** + * Function called by testbed once we are connected to stats + * service. Get the statistics for the services of interest. + * + * @param cls the 'struct RegexPeer' for which we connected to stats + * @param op connect operation handle + * @param ca_result handle to stats service + * @param emsg error message on failure + */ +static void +stats_connect_cb (void *cls, + struct GNUNET_TESTBED_Operation *op, + void *ca_result, + const char *emsg); + + +/** + * Start announcing the next regex in the DHT. + * + * @param cls Index of the next peer in the peers array. + */ +static void +announce_next_regex (void *cls); + + +/******************************************************************************/ +/******************************** SHUTDOWN **********************************/ +/******************************************************************************/ + + +/** + * Shutdown nicely + * + * @param cls NULL + */ +static void +do_shutdown (void *cls) +{ + struct RegexPeer *peer; + unsigned int peer_cnt; + unsigned int search_str_cnt; + char output_buffer[512]; + size_t size; + + if (NULL != abort_task) + { + GNUNET_SCHEDULER_cancel (abort_task); + abort_task = NULL; + } + if (NULL != register_hosts_task) + { + GNUNET_SCHEDULER_cancel (register_hosts_task); + register_hosts_task = NULL; + } + for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++) + { + peer = &peers[peer_cnt]; + + if (GNUNET_YES != peer->search_str_matched && NULL != data_file) + { + prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time); + size = + GNUNET_snprintf (output_buffer, + sizeof (output_buffer), + "%p Search string not found: %s (%d)\n" + "%p On peer: %u (%p)\n" + "%p After: %s\n", + peer, peer->search_str, peer->search_str_matched, + peer, peer->id, peer, + peer, + GNUNET_STRINGS_relative_time_to_string (prof_time, + GNUNET_NO)); + if (size != GNUNET_DISK_file_write (data_file, output_buffer, size)) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n"); + } + + if (NULL != peers[peer_cnt].op_handle) + GNUNET_TESTBED_operation_done (peers[peer_cnt].op_handle); + } + + if (NULL != data_file) + { + GNUNET_DISK_file_close (data_file); + data_file = NULL; + } + for (search_str_cnt = 0; + search_str_cnt < num_peers && NULL != search_strings; + search_str_cnt++) + { + GNUNET_free_non_null (search_strings[search_str_cnt]); + } + GNUNET_free_non_null (search_strings); + search_strings = NULL; + + if (NULL != reg_handle) + { + GNUNET_TESTBED_cancel_registration (reg_handle); + reg_handle = NULL; + } + if (NULL != mc) + { + GNUNET_TESTBED_controller_disconnect (mc); + mc = NULL; + } + if (NULL != mc_proc) + { + GNUNET_TESTBED_controller_stop (mc_proc); + mc_proc = NULL; + } + if (NULL != cfg) + { + GNUNET_CONFIGURATION_destroy (cfg); + cfg = NULL; + } +} + + +/** + * abort task to run on test timed out + * + * @param cls NULL + */ +static void +do_abort (void *cls) +{ + unsigned long i = (unsigned long) cls; + + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Aborting from line %lu...\n", i); + abort_task = NULL; + result = GNUNET_SYSERR; + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); +} + + +/******************************************************************************/ +/********************* STATISTICS SERVICE CONNECTIONS ***********************/ +/******************************************************************************/ + +/** + * Adapter function called to establish a connection to + * statistics service. + * + * @param cls closure + * @param cfg configuration of the peer to connect to; will be available until + * GNUNET_TESTBED_operation_done() is called on the operation returned + * from GNUNET_TESTBED_service_connect() + * @return service handle to return in 'op_result', NULL on error + */ +static void * +stats_ca (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + return GNUNET_STATISTICS_create ("", cfg); +} + + +/** + * Adapter function called to destroy a connection to + * statistics service. + * + * @param cls closure + * @param op_result service handle returned from the connect adapter + */ +static void +stats_da (void *cls, void *op_result) +{ + struct RegexPeer *peer = cls; + + GNUNET_assert (op_result == peer->stats_handle); + + GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO); + peer->stats_handle = NULL; +} + + +/** + * Process statistic values. Write all values to global 'data_file', if present. + * + * @param cls closure + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration + */ +static int +stats_iterator (void *cls, + const char *subsystem, + const char *name, + uint64_t value, int is_persistent) +{ + struct RegexPeer *peer = cls; + char output_buffer[512]; + size_t size; + + if (NULL == data_file) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "%p -> %s [%s]: %llu\n", + peer, + subsystem, + name, + (unsigned long long) value); + return GNUNET_OK; + } + size = + GNUNET_snprintf (output_buffer, + sizeof (output_buffer), + "%p [%s] %llu %s\n", + peer, + subsystem, value, name); + if (size != GNUNET_DISK_file_write (data_file, output_buffer, size)) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Unable to write to file!\n"); + + return GNUNET_OK; +} + + +/** + * Stats callback. Finish the stats testbed operation and when all stats have + * been iterated, shutdown the profiler. + * + * @param cls closure + * @param success GNUNET_OK if statistics were + * successfully obtained, GNUNET_SYSERR if not. + */ +static void +stats_cb (void *cls, + int success) +{ + static unsigned int peer_cnt; + struct RegexPeer *peer = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Getting statistics for peer %u failed!\n", + peer->id); + return; + } + + GNUNET_assert (NULL != peer->op_handle); + + GNUNET_TESTBED_operation_done (peer->op_handle); + peer->op_handle = NULL; + + peer_cnt++; + peer = &peers[peer_cnt]; + + fprintf (stderr, "s"); + if (peer_cnt == num_peers) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "\nCollecting stats finished. Shutting down.\n"); + GNUNET_SCHEDULER_shutdown (); + result = GNUNET_OK; + } + else + { + peer->op_handle = + GNUNET_TESTBED_service_connect (NULL, + peer->peer_handle, + "statistics", + &stats_connect_cb, + peer, + &stats_ca, + &stats_da, + peer); + } +} + + +/** + * Function called by testbed once we are connected to stats + * service. Get the statistics for the services of interest. + * + * @param cls the 'struct RegexPeer' for which we connected to stats + * @param op connect operation handle + * @param ca_result handle to stats service + * @param emsg error message on failure + */ +static void +stats_connect_cb (void *cls, + struct GNUNET_TESTBED_Operation *op, + void *ca_result, + const char *emsg) +{ + struct RegexPeer *peer = cls; + + if (NULL == ca_result || NULL != emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to connect to statistics service on peer %u: %s\n", + peer->id, emsg); + + peer->stats_handle = NULL; + return; + } + + peer->stats_handle = ca_result; + + if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL, + &stats_cb, + &stats_iterator, peer)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not get statistics of peer %u!\n", peer->id); + } +} + + +/** + * Task to collect all statistics from all peers, will shutdown the + * profiler, when done. + * + * @param cls NULL + */ +static void +do_collect_stats (void *cls) +{ + struct RegexPeer *peer = &peers[0]; + + GNUNET_assert (NULL != peer->peer_handle); + + peer->op_handle = + GNUNET_TESTBED_service_connect (NULL, + peer->peer_handle, + "statistics", + &stats_connect_cb, + peer, + &stats_ca, + &stats_da, + peer); +} + + +/******************************************************************************/ +/************************ REGEX FIND CONNECTIONS **************************/ +/******************************************************************************/ + + +/** + * Start searching for the next string in the DHT. + * + * @param cls Index of the next peer in the peers array. + */ +static void +find_string (void *cls); + + +/** + * Method called when we've found a peer that announced a regex + * that matches our search string. Now get the statistics. + * + * @param cls Closure provided in REGEX_INTERNAL_search. + * @param id Peer providing a regex that matches the string. + * @param get_path Path of the get request. + * @param get_path_length Lenght of get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the put_path. + */ +static void +regex_found_handler (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length) +{ + struct RegexPeer *peer = cls; + char output_buffer[512]; + size_t size; + + if (GNUNET_YES == peer->search_str_matched) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "String %s on peer %u already matched!\n", + peer->search_str, peer->id); + return; + } + + strings_found++; + parallel_searches--; + + if (NULL != peer->timeout) + { + GNUNET_SCHEDULER_cancel (peer->timeout); + peer->timeout = NULL; + if (GNUNET_NO == in_shutdown) + GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); + } + + if (NULL == id) + { + // FIXME not possible right now + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "String matching timed out for string %s on peer %u (%i/%i)\n", + peer->search_str, peer->id, strings_found, num_peers); + peer->search_str_matched = GNUNET_SYSERR; + } + else + { + prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "String %s found on peer %u after %s (%i/%i) (%u||)\n", + peer->search_str, peer->id, + GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO), + strings_found, num_peers, parallel_searches); + + peer->search_str_matched = GNUNET_YES; + + if (NULL != data_file) + { + size = + GNUNET_snprintf (output_buffer, + sizeof (output_buffer), + "%p Peer: %u\n" + "%p Search string: %s\n" + "%p Search duration: %s\n\n", + peer, peer->id, + peer, peer->search_str, + peer, + GNUNET_STRINGS_relative_time_to_string (prof_time, + GNUNET_NO)); + + if (size != GNUNET_DISK_file_write (data_file, output_buffer, size)) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n"); + } + } + + GNUNET_TESTBED_operation_done (peer->op_handle); + peer->op_handle = NULL; + + if (strings_found == num_peers) + { + prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "All strings successfully matched in %s\n", + GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO)); + + if (NULL != search_timeout_task) + { + GNUNET_SCHEDULER_cancel (search_timeout_task); + search_timeout_task = NULL; + } + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n"); + GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL); + } +} + + +/** + * Connect by string timeout task. This will cancel the profiler after the + * specified timeout 'search_timeout'. + * + * @param cls NULL + */ +static void +search_timed_out (void *cls) +{ + unsigned int i; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Finding matches to all strings did not succeed after %s.\n", + GNUNET_STRINGS_relative_time_to_string (search_timeout_time, + GNUNET_NO)); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Found %i of %i strings\n", strings_found, num_peers); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Search timed out after %s." + "Collecting stats and shutting down.\n", + GNUNET_STRINGS_relative_time_to_string (search_timeout_time, + GNUNET_NO)); + + in_shutdown = GNUNET_YES; + for (i = 0; i < num_peers; i++) + { + if (NULL != peers[i].op_handle) + { + GNUNET_TESTBED_operation_done (peers[i].op_handle); + peers[i].op_handle = NULL; + } + } + GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL); +} + + +/** + * Search timed out. It might still complete in the future, + * but we should start another one. + * + * @param cls Index of the next peer in the peers array. + */ +static void +find_timed_out (void *cls) +{ + struct RegexPeer *p = cls; + + p->timeout = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Searching for string \"%s\" on peer %d timed out.\n", + p->search_str, + p->id); + if (GNUNET_NO == in_shutdown) + GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); +} + + +/** + * Start searching for a string in the DHT. + * + * @param cls Index of the next peer in the peers array. + */ +static void +find_string (void *cls) +{ + unsigned int search_peer = (unsigned int) (long) cls; + + if ( (search_peer >= num_peers) || + (GNUNET_YES == in_shutdown) ) + return; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Searching for string \"%s\" on peer %d (%u||)\n", + peers[search_peer].search_str, + search_peer, + parallel_searches); + + peers[search_peer].op_handle = + GNUNET_TESTBED_service_connect (NULL, + peers[search_peer].peer_handle, + "dht", + &dht_connect_cb, + &peers[search_peer], + &dht_ca, + &dht_da, + &peers[search_peer]); + GNUNET_assert (NULL != peers[search_peer].op_handle); + peers[search_peer].timeout + = GNUNET_SCHEDULER_add_delayed (FIND_TIMEOUT, + &find_timed_out, + &peers[search_peer]); +} + + +/** + * Callback called when testbed has started the daemon we asked for. + * + * @param cls NULL + * @param op the operation handle + * @param emsg NULL on success; otherwise an error description + */ +static void +daemon_started (void *cls, + struct GNUNET_TESTBED_Operation *op, + const char *emsg) +{ + struct RegexPeer *peer = (struct RegexPeer *) cls; + unsigned long search_peer; + unsigned int i; + + GNUNET_TESTBED_operation_done (peer->daemon_op); + peer->daemon_op = NULL; + if (NULL != emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg); + GNUNET_assert (0); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Deamon %u started successfully\n", peer->id); + } + + /* Find a peer to look for a string matching the regex announced */ + search_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + num_peers); + for (i = 0; peers[search_peer].search_str != NULL; i++) + { + search_peer = (search_peer + 1) % num_peers; + if (i > num_peers) + GNUNET_assert (0); /* we ran out of peers, must be a bug */ + } + peers[search_peer].search_str = search_strings[peer->id]; + peers[search_peer].search_str_matched = GNUNET_NO; + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_saturating_multiply( + reannounce_period_max, + 2), + &find_string, + (void *) search_peer); +} + + +/** + * Task to start the daemons on each peer so that the regexes are announced + * into the DHT. + * + * @param cls NULL + * @param tc the task context + */ +static void +do_announce (void *cls) +{ + unsigned int i; + + if (GNUNET_YES == in_shutdown) + return; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Starting announce.\n"); + for (i = 0; i < init_parallel_searches; i++) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + " scheduling announce %u\n", + i); + (void) GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); + } +} + + +/** + * Start announcing the next regex in the DHT. + * + * @param cls Closure (unused). + */ +static void +announce_next_regex (void *cls) +{ + struct RegexPeer *peer; + + if (GNUNET_YES == in_shutdown) + return; + if (next_search >= num_peers) + { + if (strings_found != num_peers) + { + struct GNUNET_TIME_Relative new_delay; + if (NULL != search_timeout_task) + GNUNET_SCHEDULER_cancel (search_timeout_task); + new_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15); + search_timeout_task = GNUNET_SCHEDULER_add_delayed (new_delay, + &search_timed_out, + NULL); + } + return; + } + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search); + peer = &peers[next_search]; + peer->daemon_op = + GNUNET_TESTBED_peer_manage_service (NULL, + peer->peer_handle, + "regexprofiler", + &daemon_started, + peer, + 1); + next_search++; + parallel_searches++; +} + + +/** + * DHT connect callback. Called when we are connected to the dht service for + * the peer in 'cls'. If successfull we connect to the stats service of this + * peer and then try to match the search string of this peer. + * + * @param cls internal peer id. + * @param op operation handle. + * @param ca_result connect adapter result. + * @param emsg error message. + */ +static void +dht_connect_cb (void *cls, + struct GNUNET_TESTBED_Operation *op, + void *ca_result, + const char *emsg) +{ + struct RegexPeer *peer = (struct RegexPeer *) cls; + + if (NULL != emsg || NULL == op || NULL == ca_result) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg); + GNUNET_assert (0); + } + + GNUNET_assert (NULL != peer->dht_handle); + GNUNET_assert (peer->op_handle == op); + GNUNET_assert (peer->dht_handle == ca_result); + + peer->search_str_matched = GNUNET_NO; + peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle, + peer->search_str, + ®ex_found_handler, peer, + NULL); + peer->prof_start_time = GNUNET_TIME_absolute_get (); +} + + +/** + * DHT connect adapter. Opens a connection to the dht service. + * + * @param cls Closure (peer). + * @param cfg Configuration handle. + * + * @return + */ +static void * +dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct RegexPeer *peer = cls; + + peer->dht_handle = GNUNET_DHT_connect (cfg, 32); + + return peer->dht_handle; +} + + +/** + * Adapter function called to destroy a connection to the dht service. + * + * @param cls Closure (peer). + * @param op_result Service handle returned from the connect adapter. + */ +static void +dht_da (void *cls, void *op_result) +{ + struct RegexPeer *peer = (struct RegexPeer *) cls; + + GNUNET_assert (peer->dht_handle == op_result); + + if (NULL != peer->search_handle) + { + REGEX_INTERNAL_search_cancel (peer->search_handle); + peer->search_handle = NULL; + } + + if (NULL != peer->dht_handle) + { + GNUNET_DHT_disconnect (peer->dht_handle); + peer->dht_handle = NULL; + } +} + + +/** + * Signature of a main function for a testcase. + * + * @param cls NULL + * @param h the run handle + * @param num_peers_ number of peers in 'peers' + * @param testbed_peers handle to peers run in the testbed. NULL upon timeout (see + * GNUNET_TESTBED_test_run()). + * @param links_succeeded the number of overlay link connection attempts that + * succeeded + * @param links_failed the number of overlay link connection attempts that + * failed + */ +static void +test_master (void *cls, + struct GNUNET_TESTBED_RunHandle *h, + unsigned int num_peers_, + struct GNUNET_TESTBED_Peer **testbed_peers, + unsigned int links_succeeded, + unsigned int links_failed) +{ + unsigned int i; + + GNUNET_assert (num_peers_ == num_peers); + + prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Testbed started in %s\n", + GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO)); + + if (NULL != abort_task) + { + GNUNET_SCHEDULER_cancel (abort_task); + abort_task = NULL; + } + + for (i = 0; i < num_peers; i++) + { + peers[i].peer_handle = testbed_peers[i]; + } + if (GNUNET_NO == + GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT")) + { + struct GNUNET_TIME_Relative settle_time; + + settle_time = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, + 10 * num_peers); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Waiting for DHT for %s to settle new connections.\n\n", + GNUNET_STRINGS_relative_time_to_string(settle_time, GNUNET_NO)); + GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL); + } + else + { + GNUNET_SCHEDULER_add_now (&do_announce, NULL); + } + search_timeout_task = + GNUNET_SCHEDULER_add_delayed (search_timeout_time, &search_timed_out, NULL); +} + +/** + * Function that will be called whenever something in the testbed changes. + * + * @param cls closure, NULL + * @param event information on what is happening + */ +static void +master_controller_cb (void *cls, + const struct GNUNET_TESTBED_EventInformation *event) +{ + switch (event->type) + { + case GNUNET_TESTBED_ET_CONNECT: + printf("."); + break; + case GNUNET_TESTBED_ET_PEER_START: + printf("#"); + break; + default: + break; + } + fflush(stdout); +} + + +/******************************************************************************/ +/*************************** TESTBED PEER SETUP *****************************/ +/******************************************************************************/ + +/** + * Process the text buffer counting the non-empty lines and separating them + * with NULL characters, for later ease of copy using (as)printf. + * + * @param data Memory buffer with strings. + * @param data_size Size of the @a data buffer in bytes. + * @param str_max Maximum number of strings to return. + * @return Positive number of lines found in the buffer, + * #GNUNET_SYSERR otherwise. + */ +static int +count_and_separate_strings (char *data, + uint64_t data_size, + unsigned int str_max) +{ + char *buf; // Keep track of last string to skip blank lines + unsigned int offset; + unsigned int str_cnt; + + buf = data; + offset = 0; + str_cnt = 0; + while ( (offset < (data_size - 1)) && (str_cnt < str_max) ) + { + offset++; + if ( ((data[offset] == '\n')) && + (buf != &data[offset]) ) + { + data[offset] = '\0'; + str_cnt++; + buf = &data[offset + 1]; + } + else if ( (data[offset] == '\n') || + (data[offset] == '\0') ) + buf = &data[offset + 1]; + } + return str_cnt; +} + + +/** + * Allocate a string array and fill it with the prefixed strings + * from a pre-processed, NULL-separated memory region. + * + * @param data Preprocessed memory with strings + * @param data_size Size of the @a data buffer in bytes. + * @param strings Address of the string array to be created. + * Must be freed by caller if function end in success. + * @param str_cnt String count. The @a data buffer should contain + * at least this many NULL-separated strings. + * @return #GNUNET_OK in ase of success, #GNUNET_SYSERR otherwise. + * In case of error @a strings must not be freed. + */ +static int +create_string_array (char *data, uint64_t data_size, + char ***strings, unsigned int str_cnt) +{ + uint64_t offset; + uint64_t len; + unsigned int i; + + *strings = GNUNET_malloc (sizeof (char *) * str_cnt); + offset = 0; + for (i = 0; i < str_cnt; i++) + { + len = strlen (&data[offset]); + if (offset + len >= data_size) + { + GNUNET_free (*strings); + *strings = NULL; + return GNUNET_SYSERR; + } + if (0 == len) // empty line + { + offset++; + i--; + continue; + } + + GNUNET_asprintf (&(*strings)[i], + "%s%s", + regex_prefix, + &data[offset]); + offset += len + 1; + } + return GNUNET_OK; +} + + +/** + * Load search strings from given filename. One search string per line. + * + * @param filename filename of the file containing the search strings. + * @param strings set of strings loaded from file. Caller needs to free this + * if number returned is greater than zero. + * @param limit upper limit on the number of strings read from the file + * @return number of strings found in the file. #GNUNET_SYSERR on error. + */ +static int +load_search_strings (const char *filename, + char ***strings, + unsigned int limit) +{ + char *data; + uint64_t filesize; + int str_cnt; + + /* Sanity checks */ + if (NULL == filename) + { + return GNUNET_SYSERR; + } + if (GNUNET_YES != GNUNET_DISK_file_test (filename)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not find search strings file %s\n", filename); + return GNUNET_SYSERR; + } + if (GNUNET_OK != + GNUNET_DISK_file_size (filename, + &filesize, + GNUNET_YES, + GNUNET_YES)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Search strings file %s cannot be read.\n", + filename); + return GNUNET_SYSERR; + } + if (0 == filesize) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Search strings file %s is empty.\n", + filename); + return GNUNET_SYSERR; + } + + /* Read data into memory */ + data = GNUNET_malloc (filesize + 1); + if (filesize != GNUNET_DISK_fn_read (filename, + data, + filesize)) + { + GNUNET_free (data); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not read search strings file %s.\n", + filename); + return GNUNET_SYSERR; + } + + /* Process buffer and build array */ + str_cnt = count_and_separate_strings (data, filesize, limit); + if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt)) + { + str_cnt = GNUNET_SYSERR; + } + GNUNET_free (data); + return str_cnt; +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param config configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *config) +{ + unsigned int nsearchstrs; + unsigned int i; + struct GNUNET_TIME_Relative abort_time; + + in_shutdown = GNUNET_NO; + + /* Check config */ + if (NULL == config) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No configuration file given. Exiting\n")); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + cfg = GNUNET_CONFIGURATION_dup (config); + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER", + "REGEX_PREFIX", + ®ex_prefix)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, + "regexprofiler", + "regex_prefix"); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER", + "PARALLEL_SEARCHES", + &init_parallel_searches)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Configuration option \"PARALLEL_SEARCHES\" missing." + " Using default (%d)\n", 10); + init_parallel_searches = 10; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER", + "REANNOUNCE_PERIOD_MAX", + &reannounce_period_max)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "reannounce_period_max not given. Using 10 minutes.\n"); + reannounce_period_max = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10); + } + + /* Check arguments */ + if (NULL == policy_dir) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No policy directory specified on command line. Exiting.\n")); + return; + } + if (GNUNET_YES != GNUNET_DISK_directory_test (policy_dir, GNUNET_YES)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Specified policies directory does not exist. Exiting.\n")); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL, NULL))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No files found in `%s'\n"), + policy_dir); + return; + } + GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER", + "POLICY_DIR", policy_dir); + if (GNUNET_YES != GNUNET_DISK_file_test (strings_file)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No search strings file given. Exiting.\n")); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + nsearchstrs = load_search_strings (strings_file, + &search_strings, + num_peers); + if (num_peers != nsearchstrs) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error loading search strings.\n"); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "File (%s) does not contain enough strings (%u/%u).\n", + strings_file, nsearchstrs, num_peers); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + if ( (0 == num_peers) || (NULL == search_strings)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Error loading search strings. Exiting.\n")); + GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); + return; + } + for (i = 0; i < num_peers; i++) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "search string: %s\n", + search_strings[i]); + + /* Check logfile */ + if ( (NULL != data_filename) && + (NULL == (data_file = + GNUNET_DISK_file_open (data_filename, + GNUNET_DISK_OPEN_READWRITE | + GNUNET_DISK_OPEN_TRUNCATE | + GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE))) ) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, + "open", + data_filename); + return; + } + + /* Initialize peers */ + peers = GNUNET_malloc (sizeof (struct RegexPeer) * num_peers); + for (i = 0; i < num_peers; i++) + peers[i].id = i; + + GNUNET_CONFIGURATION_set_value_number (cfg, + "TESTBED", "OVERLAY_RANDOM_LINKS", + num_peers * 20); + GNUNET_CONFIGURATION_set_value_number (cfg, + "DHT", "FORCE_NSE", + (long long unsigned) + (log (num_peers) / log (2.0))); + event_mask = 0LL; +/* For feedback about the start process activate these and pass master_cb */ + event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_START); +// event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP); + event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT); +// event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT); + prof_start_time = GNUNET_TIME_absolute_get (); + GNUNET_TESTBED_run (hosts_file, + cfg, + num_peers, + event_mask, + &master_controller_cb, + NULL, /* master_controller_cb cls */ + &test_master, + NULL); /* test_master cls */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED", + "SETUP_TIMEOUT", + &abort_time)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "SETUP_TIMEOUT not given. Using 15 minutes.\n"); + abort_time = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15); + } + abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES); + abort_task = + GNUNET_SCHEDULER_add_delayed (abort_time, + &do_abort, + (void*) __LINE__); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "setup_timeout: %s\n", + GNUNET_STRINGS_relative_time_to_string (abort_time, GNUNET_YES)); +} + + +/** + * Main function. + * + * @param argc argument count + * @param argv argument values + * @return 0 on success + */ +int +main (int argc, char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + + GNUNET_GETOPT_option_filename ('o', + "output-file", + "FILENAME", + gettext_noop ("name of the file for writing statistics"), + &data_filename), + + GNUNET_GETOPT_option_relative_time ('t', + "matching-timeout", + "TIMEOUT", + gettext_noop ("wait TIMEOUT before ending the experiment"), + &search_timeout_time), + + GNUNET_GETOPT_option_filename ('p', + "policy-dir", + "DIRECTORY", + gettext_noop ("directory with policy files"), + &policy_dir), + + + GNUNET_GETOPT_option_filename ('s', + "strings-file", + "FILENAME", + gettext_noop ("name of file with input strings"), + &strings_file), + + GNUNET_GETOPT_option_filename ('H', + "hosts-file", + "FILENAME", + gettext_noop ("name of file with hosts' names"), + &hosts_file), + + GNUNET_GETOPT_OPTION_END + }; + int ret; + + if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) + return 2; + result = GNUNET_SYSERR; + ret = + GNUNET_PROGRAM_run (argc, argv, + "gnunet-regex-profiler", + _("Profiler for regex"), + options, &run, NULL); + if (GNUNET_OK != ret) + return ret; + if (GNUNET_OK != result) + return 1; + return 0; +} diff --git a/src/regex/gnunet-regex-simulation-profiler.c b/src/regex/gnunet-regex-simulation-profiler.c new file mode 100644 index 000000000..b7e256f48 --- /dev/null +++ b/src/regex/gnunet-regex-simulation-profiler.c @@ -0,0 +1,723 @@ +/* + This file is part of GNUnet. + Copyright (C) 2011, 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + + +/** + * @file regex/gnunet-regex-simulation-profiler.c + * @brief Regex profiler that dumps all DFAs into a database instead of + * using the DHT (with cadet). + * @author Maximilian Szengel + * @author Christophe Genevey + * + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "regex_internal_lib.h" +#include "gnunet_mysql_lib.h" +#include "gnunet_my_lib.h" +#include + +/** + * MySQL statement to insert an edge. + */ +#define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` "\ + "(`key`, `label`, `to_key`, `accepting`) "\ + "VALUES (?, ?, ?, ?);" + +/** + * MySQL statement to select a key count. + */ +#define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` "\ + "WHERE `key` = ? AND `label` = ?;" + +/** + * Simple struct to keep track of progress, and print a + * nice little percentage meter for long running tasks. + */ +struct ProgressMeter +{ + /** + * Total number of elements. + */ + unsigned int total; + + /** + * Intervall for printing percentage. + */ + unsigned int modnum; + + /** + * Number of dots to print. + */ + unsigned int dotnum; + + /** + * Completed number. + */ + unsigned int completed; + + /** + * Should the meter be printed? + */ + int print; + + /** + * String to print on startup. + */ + char *startup_string; +}; + + +/** + * Handle for the progress meter + */ +static struct ProgressMeter *meter; + +/** + * Scan task identifier; + */ +static struct GNUNET_SCHEDULER_Task *scan_task; + +/** + * Global testing status. + */ +static int result; + +/** + * MySQL context. + */ +static struct GNUNET_MYSQL_Context *mysql_ctx; + +/** + * MySQL prepared statement handle. + */ +static struct GNUNET_MYSQL_StatementHandle *stmt_handle; + +/** + * MySQL prepared statement handle for `key` select. + */ +static struct GNUNET_MYSQL_StatementHandle *select_stmt_handle; + +/** + * MySQL table name. + */ +static char *table_name; + +/** + * Policy dir containing files that contain policies. + */ +static char *policy_dir; + +/** + * Number of policy files. + */ +static unsigned int num_policy_files; + +/** + * Number of policies. + */ +static unsigned int num_policies; + +/** + * Maximal path compression length. + */ +static unsigned int max_path_compression; + +/** + * Number of merged transitions. + */ +static unsigned long long num_merged_transitions; + +/** + * Number of merged states from different policies. + */ +static unsigned long long num_merged_states; + +/** + * Prefix to add before every regex we're announcing. + */ +static char *regex_prefix; + + +/** + * Create a meter to keep track of the progress of some task. + * + * @param total the total number of items to complete + * @param start_string a string to prefix the meter with (if printing) + * @param print GNUNET_YES to print the meter, GNUNET_NO to count + * internally only + * + * @return the progress meter + */ +static struct ProgressMeter * +create_meter (unsigned int total, char *start_string, int print) +{ + struct ProgressMeter *ret; + + ret = GNUNET_new (struct ProgressMeter); + ret->print = print; + ret->total = total; + ret->modnum = total / 4; + if (ret->modnum == 0) /* Divide by zero check */ + ret->modnum = 1; + ret->dotnum = (total / 50) + 1; + if (start_string != NULL) + ret->startup_string = GNUNET_strdup (start_string); + else + ret->startup_string = GNUNET_strdup (""); + + return ret; +} + + +/** + * Update progress meter (increment by one). + * + * @param meter the meter to update and print info for + * + * @return GNUNET_YES if called the total requested, + * GNUNET_NO if more items expected + */ +static int +update_meter (struct ProgressMeter *meter) +{ + if (meter->print == GNUNET_YES) + { + if (meter->completed % meter->modnum == 0) + { + if (meter->completed == 0) + { + FPRINTF (stdout, "%sProgress: [0%%", meter->startup_string); + } + else + FPRINTF (stdout, "%d%%", + (int) (((float) meter->completed / meter->total) * 100)); + } + else if (meter->completed % meter->dotnum == 0) + FPRINTF (stdout, "%s", "."); + + if (meter->completed + 1 == meter->total) + FPRINTF (stdout, "%d%%]\n", 100); + fflush (stdout); + } + meter->completed++; + + if (meter->completed == meter->total) + return GNUNET_YES; + if (meter->completed > meter->total) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n"); + return GNUNET_NO; +} + + +/** + * Reset progress meter. + * + * @param meter the meter to reset + * + * @return #GNUNET_YES if meter reset, + * #GNUNET_SYSERR on error + */ +static int +reset_meter (struct ProgressMeter *meter) +{ + if (meter == NULL) + return GNUNET_SYSERR; + + meter->completed = 0; + return GNUNET_YES; +} + + +/** + * Release resources for meter + * + * @param meter the meter to free + */ +static void +free_meter (struct ProgressMeter *meter) +{ + GNUNET_free_non_null (meter->startup_string); + GNUNET_free (meter); +} + + +/** + * Shutdown task. + * + * @param cls NULL + */ +static void +do_shutdown (void *cls) +{ + if (NULL != mysql_ctx) + { + GNUNET_MYSQL_context_destroy (mysql_ctx); + mysql_ctx = NULL; + } + if (NULL != meter) + { + free_meter (meter); + meter = NULL; + } +} + + +/** + * Abort task to run on test timed out. + * + * FIXME: this doesn't actually work, it used to cancel + * the already running 'scan_task', but now that should + * always be NULL and do nothing. We instead need to set + * a global variable and abort scan_task internally, not + * via scheduler. + * + * @param cls NULL + */ +static void +do_abort (void *cls) +{ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n"); + if (NULL != scan_task) + { + GNUNET_SCHEDULER_cancel (scan_task); + scan_task = NULL; + } + result = GNUNET_SYSERR; + GNUNET_SCHEDULER_shutdown (); +} + +/** + * Iterator over all states that inserts each state into the MySQL db. + * + * @param cls closure. + * @param key hash for current state. + * @param proof proof for current state. + * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not. + * @param num_edges number of edges leaving current state. + * @param edges edges leaving current state. + */ +static void +regex_iterator (void *cls, + const struct GNUNET_HashCode *key, + const char *proof, + int accepting, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges) +{ + unsigned int i; + int result; + + uint32_t iaccepting = (uint32_t)accepting; + uint64_t total; + + GNUNET_assert (NULL != mysql_ctx); + + for (i = 0; i < num_edges; i++) + { + struct GNUNET_MY_QueryParam params_select[] = { + GNUNET_MY_query_param_auto_from_type (key), + GNUNET_MY_query_param_string (edges[i].label), + GNUNET_MY_query_param_end + }; + + struct GNUNET_MY_ResultSpec results_select[] = { + GNUNET_MY_result_spec_uint64 (&total), + GNUNET_MY_result_spec_end + }; + + result = + GNUNET_MY_exec_prepared (mysql_ctx, + select_stmt_handle, + params_select); + + if (GNUNET_SYSERR == result) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error executing prepared mysql select statement\n"); + GNUNET_SCHEDULER_add_now (&do_abort, NULL); + return; + } + + result = + GNUNET_MY_extract_result (select_stmt_handle, + results_select); + + if (GNUNET_SYSERR == result) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error extracting result mysql select statement\n"); + GNUNET_SCHEDULER_add_now (&do_abort, NULL); + return; + } + + if (-1 != total && total > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n", + (unsigned long long)total, + GNUNET_h2s (key), edges[i].label); + } + + struct GNUNET_MY_QueryParam params_stmt[] = { + GNUNET_MY_query_param_auto_from_type (&key), + GNUNET_MY_query_param_string (edges[i].label), + GNUNET_MY_query_param_auto_from_type (&edges[i].destination), + GNUNET_MY_query_param_uint32 (&iaccepting), + GNUNET_MY_query_param_end + }; + + result = + GNUNET_MY_exec_prepared (mysql_ctx, + stmt_handle, + params_stmt); + + if (0 == result) + { + char *key_str = GNUNET_strdup (GNUNET_h2s (key)); + char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination)); + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n", + key_str, + edges[i].label, + to_key_str, + accepting); + + GNUNET_free (key_str); + GNUNET_free (to_key_str); + num_merged_transitions++; + } + else if (-1 != total) + { + num_merged_states++; + } + + if (GNUNET_SYSERR == result || (1 != result && 0 != result)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n", + result); + GNUNET_SCHEDULER_add_now (&do_abort, NULL); + } + } + + if (0 == num_edges) + { + struct GNUNET_MY_QueryParam params_stmt[] = { + GNUNET_MY_query_param_auto_from_type (key), + GNUNET_MY_query_param_string (""), + GNUNET_MY_query_param_fixed_size (NULL, 0), + GNUNET_MY_query_param_uint32 (&iaccepting), + GNUNET_MY_query_param_end + }; + + result = + GNUNET_MY_exec_prepared (mysql_ctx, + stmt_handle, + params_stmt); + + if (1 != result && 0 != result) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n", + result); + GNUNET_SCHEDULER_add_now (&do_abort, NULL); + } + } +} + + +/** + * Announce a regex by creating the DFA and iterating over each state, inserting + * each state into a MySQL database. + * + * @param regex regular expression. + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure. + */ +static int +announce_regex (const char *regex) +{ + struct REGEX_INTERNAL_Automaton *dfa; + + dfa = + REGEX_INTERNAL_construct_dfa (regex, + strlen (regex), + max_path_compression); + + if (NULL == dfa) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to create DFA for regex %s\n", + regex); + GNUNET_SCHEDULER_add_now (&do_abort, NULL); + return GNUNET_SYSERR; + } + REGEX_INTERNAL_iterate_all_edges (dfa, + ®ex_iterator, NULL); + REGEX_INTERNAL_automaton_destroy (dfa); + + return GNUNET_OK; +} + + +/** + * Function called with a filename. + * + * @param cls closure + * @param filename complete filename (absolute path) + * @return #GNUNET_OK to continue to iterate, + * #GNUNET_SYSERR to abort iteration with error! + */ +static int +policy_filename_cb (void *cls, const char *filename) +{ + char *regex; + char *data; + char *buf; + uint64_t filesize; + unsigned int offset; + + GNUNET_assert (NULL != filename); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Announcing regexes from file %s\n", + filename); + + if (GNUNET_YES != GNUNET_DISK_file_test (filename)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not find policy file %s\n", + filename); + return GNUNET_OK; + } + if (GNUNET_OK != + GNUNET_DISK_file_size (filename, &filesize, + GNUNET_YES, GNUNET_YES)) + filesize = 0; + if (0 == filesize) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n", + filename); + return GNUNET_OK; + } + data = GNUNET_malloc (filesize); + if (filesize != GNUNET_DISK_fn_read (filename, data, filesize)) + { + GNUNET_free (data); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not read policy file %s.\n", + filename); + return GNUNET_OK; + } + + update_meter (meter); + + buf = data; + offset = 0; + regex = NULL; + while (offset < (filesize - 1)) + { + offset++; + if (((data[offset] == '\n')) && (buf != &data[offset])) + { + data[offset] = '|'; + num_policies++; + buf = &data[offset + 1]; + } + else if ((data[offset] == '\n') || (data[offset] == '\0')) + buf = &data[offset + 1]; + } + data[offset] = '\0'; + GNUNET_asprintf (®ex, "%s(%s)", regex_prefix, data); + GNUNET_assert (NULL != regex); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Announcing regex: %s\n", regex); + + if (GNUNET_OK != announce_regex (regex)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not announce regex %s\n", + regex); + } + GNUNET_free (regex); + GNUNET_free (data); + return GNUNET_OK; +} + + +/** + * Iterate over files contained in policy_dir. + * + * @param cls NULL + */ +static void +do_directory_scan (void *cls) +{ + struct GNUNET_TIME_Absolute start_time; + struct GNUNET_TIME_Relative duration; + char *stmt; + + /* Create an MySQL prepared statement for the inserts */ + scan_task = NULL; + GNUNET_asprintf (&stmt, INSERT_EDGE_STMT, table_name); + stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt); + GNUNET_free (stmt); + + GNUNET_asprintf (&stmt, SELECT_KEY_STMT, table_name); + select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt); + GNUNET_free (stmt); + + GNUNET_assert (NULL != stmt_handle); + + meter = create_meter (num_policy_files, + "Announcing policy files\n", + GNUNET_YES); + start_time = GNUNET_TIME_absolute_get (); + GNUNET_DISK_directory_scan (policy_dir, + &policy_filename_cb, + stmt_handle); + duration = GNUNET_TIME_absolute_get_duration (start_time); + reset_meter (meter); + free_meter (meter); + meter = NULL; + + printf ("Announced %u files containing %u policies in %s\n" + "Duplicate transitions: %llu\nMerged states: %llu\n", + num_policy_files, + num_policies, + GNUNET_STRINGS_relative_time_to_string (duration, GNUNET_NO), + num_merged_transitions, + num_merged_states); + result = GNUNET_OK; + GNUNET_SCHEDULER_shutdown (); +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param config configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *config) +{ + if (NULL == args[0]) + { + fprintf (stderr, + _("No policy directory specified on command line. Exiting.\n")); + result = GNUNET_SYSERR; + return; + } + if (GNUNET_YES != + GNUNET_DISK_directory_test (args[0], GNUNET_YES)) + { + fprintf (stderr, + _("Specified policies directory does not exist. Exiting.\n")); + result = GNUNET_SYSERR; + return; + } + policy_dir = args[0]; + + num_policy_files = GNUNET_DISK_directory_scan (policy_dir, + NULL, NULL); + meter = NULL; + + if (NULL == table_name) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "No table name specified, using default \"NFA\".\n"); + table_name = "NFA"; + } + + mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql"); + if (NULL == mysql_ctx) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to create mysql context\n"); + result = GNUNET_SYSERR; + return; + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (config, + "regex-mysql", + "REGEX_PREFIX", + ®ex_prefix)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, + "regex-mysql", + "REGEX_PREFIX"); + result = GNUNET_SYSERR; + return; + } + + result = GNUNET_OK; + GNUNET_SCHEDULER_add_shutdown (&do_shutdown, + NULL); + scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL); +} + + +/** + * Main function. + * + * @param argc argument count + * @param argv argument values + * @return 0 on success + */ +int +main (int argc, char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + + GNUNET_GETOPT_option_string ('t', + "table", + "TABLENAME", + gettext_noop ("name of the table to write DFAs"), + &table_name), + + GNUNET_GETOPT_option_uint ('p', + "max-path-compression", + "MAX_PATH_COMPRESSION", + gettext_noop ("maximum path compression length"), + &max_path_compression), + + GNUNET_GETOPT_OPTION_END + }; + int ret; + + if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) + return 2; + + result = GNUNET_SYSERR; + ret = + GNUNET_PROGRAM_run (argc, argv, + "gnunet-regex-simulationprofiler [OPTIONS] policy-dir", + _("Profiler for regex library"), options, &run, NULL); + if (GNUNET_OK != ret) + return ret; + if (GNUNET_OK != result) + return 1; + return 0; +} diff --git a/src/regex/gnunet-service-regex.c b/src/regex/gnunet-service-regex.c new file mode 100644 index 000000000..eb10e7c07 --- /dev/null +++ b/src/regex/gnunet-service-regex.c @@ -0,0 +1,416 @@ +/* + This file is part of GNUnet. + Copyright (C) 2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +/** + * @file regex/gnunet-service-regex.c + * @brief service to advertise capabilities described as regex and to + * lookup capabilities by regex + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "regex_internal_lib.h" +#include "regex_ipc.h" + + +/** + * Information about one of our clients. + */ +struct ClientEntry +{ + + /** + * Queue for transmissions to @e client. + */ + struct GNUNET_MQ_Handle *mq; + + /** + * Handle identifying the client. + */ + struct GNUNET_SERVICE_Client *client; + + /** + * Search handle (if this client is searching). + */ + struct REGEX_INTERNAL_Search *sh; + + /** + * Announcement handle (if this client is announcing). + */ + struct REGEX_INTERNAL_Announcement *ah; + + /** + * Refresh frequency for announcements. + */ + struct GNUNET_TIME_Relative frequency; + + /** + * Task for re-announcing. + */ + struct GNUNET_SCHEDULER_Task *refresh_task; + +}; + + +/** + * Connection to the DHT. + */ +static struct GNUNET_DHT_Handle *dht; + +/** + * Handle for doing statistics. + */ +static struct GNUNET_STATISTICS_Handle *stats; + +/** + * Private key for this peer. + */ +static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key; + + +/** + * Task run during shutdown. + * + * @param cls unused + */ +static void +cleanup_task (void *cls) +{ + GNUNET_DHT_disconnect (dht); + dht = NULL; + GNUNET_STATISTICS_destroy (stats, + GNUNET_NO); + stats = NULL; + GNUNET_free (my_private_key); + my_private_key = NULL; +} + + +/** + * Periodic task to refresh our announcement of the regex. + * + * @param cls the `struct ClientEntry *` of the client that triggered the + * announcement + */ +static void +reannounce (void *cls) +{ + struct ClientEntry *ce = cls; + + REGEX_INTERNAL_reannounce (ce->ah); + ce->refresh_task = GNUNET_SCHEDULER_add_delayed (ce->frequency, + &reannounce, + ce); +} + + +/** + * Check ANNOUNCE message. + * + * @param cls identification of the client + * @param am the actual message + * @return #GNUNET_OK if @am is well-formed + */ +static int +check_announce (void *cls, + const struct AnnounceMessage *am) +{ + struct ClientEntry *ce = cls; + const char *regex; + uint16_t size; + + size = ntohs (am->header.size) - sizeof (*am); + regex = (const char *) &am[1]; + if ('\0' != regex[size - 1]) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (NULL != ce->ah) + { + /* only one announcement per client allowed */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Handle ANNOUNCE message. + * + * @param cls identification of the client + * @param am the actual message + */ +static void +handle_announce (void *cls, + const struct AnnounceMessage *am) +{ + struct ClientEntry *ce = cls; + const char *regex; + + regex = (const char *) &am[1]; + ce->frequency = GNUNET_TIME_relative_ntoh (am->refresh_delay); + ce->refresh_task = GNUNET_SCHEDULER_add_delayed (ce->frequency, + &reannounce, + ce); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting to announce regex `%s' every %s\n", + regex, + GNUNET_STRINGS_relative_time_to_string (ce->frequency, + GNUNET_NO)); + ce->ah = REGEX_INTERNAL_announce (dht, + my_private_key, + regex, + ntohs (am->compression), + stats); + if (NULL == ce->ah) + { + GNUNET_break (0); + GNUNET_SCHEDULER_cancel (ce->refresh_task); + ce->refresh_task = NULL; + GNUNET_SERVICE_client_drop (ce->client); + return; + } + GNUNET_SERVICE_client_continue (ce->client); +} + + +/** + * Handle result, pass it back to the client. + * + * @param cls the struct ClientEntry of the client searching + * @param id Peer providing a regex that matches the string. + * @param get_path Path of the get request. + * @param get_path_length Lenght of @a get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the @a put_path. + */ +static void +handle_search_result (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length) +{ + struct ClientEntry *ce = cls; + struct GNUNET_MQ_Envelope *env; + struct ResultMessage *result; + struct GNUNET_PeerIdentity *gp; + uint16_t size; + + if ( (get_path_length >= 65536) || + (put_path_length >= 65536) || + ( (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity)) + + sizeof (struct ResultMessage) >= GNUNET_MAX_MESSAGE_SIZE) + { + GNUNET_break (0); + return; + } + size = (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity); + env = GNUNET_MQ_msg_extra (result, + size, + GNUNET_MESSAGE_TYPE_REGEX_RESULT); + result->get_path_length = htons ((uint16_t) get_path_length); + result->put_path_length = htons ((uint16_t) put_path_length); + result->id = *id; + gp = &result->id; + GNUNET_memcpy (&gp[1], + get_path, + get_path_length * sizeof (struct GNUNET_PeerIdentity)); + GNUNET_memcpy (&gp[1 + get_path_length], + put_path, + put_path_length * sizeof (struct GNUNET_PeerIdentity)); + GNUNET_MQ_send (ce->mq, + env); +} + + +/** + * Check SEARCH message. + * + * @param cls identification of the client + * @param message the actual message + */ +static int +check_search (void *cls, + const struct RegexSearchMessage *sm) +{ + struct ClientEntry *ce = cls; + const char *string; + uint16_t size; + + size = ntohs (sm->header.size) - sizeof (*sm); + string = (const char *) &sm[1]; + if ('\0' != string[size - 1]) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (NULL != ce->sh) + { + /* only one search allowed per client */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Handle SEARCH message. + * + * @param cls identification of the client + * @param message the actual message + */ +static void +handle_search (void *cls, + const struct RegexSearchMessage *sm) +{ + struct ClientEntry *ce = cls; + const char *string; + + string = (const char *) &sm[1]; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting to search for `%s'\n", + string); + ce->sh = REGEX_INTERNAL_search (dht, + string, + &handle_search_result, + ce, + stats); + if (NULL == ce->sh) + { + GNUNET_break (0); + GNUNET_SERVICE_client_drop (ce->client); + return; + } + GNUNET_SERVICE_client_continue (ce->client); +} + + +/** + * Process regex requests. + * + * @param cls closure + * @param cfg configuration to use + * @param service the initialized service + */ +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_SERVICE_Handle *service) +{ + my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (cfg); + if (NULL == my_private_key) + { + GNUNET_SCHEDULER_shutdown (); + return; + } + dht = GNUNET_DHT_connect (cfg, 1024); + if (NULL == dht) + { + GNUNET_free (my_private_key); + my_private_key = NULL; + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_SCHEDULER_add_shutdown (&cleanup_task, + NULL); + stats = GNUNET_STATISTICS_create ("regex", cfg); +} + + +/** + * Callback called when a client connects to the service. + * + * @param cls closure for the service + * @param c the new client that connected to the service + * @param mq the message queue used to send messages to the client + * @return @a c + */ +static void * +client_connect_cb (void *cls, + struct GNUNET_SERVICE_Client *c, + struct GNUNET_MQ_Handle *mq) +{ + struct ClientEntry *ce; + + ce = GNUNET_new (struct ClientEntry); + ce->client = c; + ce->mq = mq; + return ce; +} + + +/** + * Callback called when a client disconnected from the service + * + * @param cls closure for the service + * @param c the client that disconnected + * @param internal_cls should be equal to @a c + */ +static void +client_disconnect_cb (void *cls, + struct GNUNET_SERVICE_Client *c, + void *internal_cls) +{ + struct ClientEntry *ce = internal_cls; + + if (NULL != ce->refresh_task) + { + GNUNET_SCHEDULER_cancel (ce->refresh_task); + ce->refresh_task = NULL; + } + if (NULL != ce->ah) + { + REGEX_INTERNAL_announce_cancel (ce->ah); + ce->ah = NULL; + } + if (NULL != ce->sh) + { + REGEX_INTERNAL_search_cancel (ce->sh); + ce->sh = NULL; + } + GNUNET_free (ce); +} + + +/** + * Define "main" method using service macro. + */ +GNUNET_SERVICE_MAIN +("regex", + GNUNET_SERVICE_OPTION_NONE, + &run, + &client_connect_cb, + &client_disconnect_cb, + NULL, + GNUNET_MQ_hd_var_size (announce, + GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE, + struct AnnounceMessage, + NULL), + GNUNET_MQ_hd_var_size (search, + GNUNET_MESSAGE_TYPE_REGEX_SEARCH, + struct RegexSearchMessage, + NULL), + GNUNET_MQ_handler_end ()); + + +/* end of gnunet-service-regex.c */ diff --git a/src/regex/perf-data.tar.gz b/src/regex/perf-data.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9e909e58e8016a6bb61c9da4d314033a9eb82a37 GIT binary patch literal 22848 zcmXV%by!r-`~KgGgwoP2pdgY^LT&TaUOJ+{b=yG8v1zbVjU~WhP-3F?U$nByZ!nR z->?53w{oRk#zreCUP!!3pv}}pJPO7o2%DK{n!1Ce#uTLA8>(Orieg@jWxkM*QtP1`wvtRxUa{!DeDArZ1-NBbU$50dOOK!ve_@BCl80X(XH|iX4!1L(=&3tPy+i9pqhdq zbW;8@6ZfkzkbuFfgQz*@N|Yj1Mc?1uIy3FC^|4aS7GEa_Voz+Eec>nk4#TS2^3;20 zo=WqF!Y9mFL~+QG_M|N8?G!AF#7vvTT43Tksuu>{c>uT03iw&~{}u8KoWN#5eqIp$ zL5%wO#ZkI>JHUKTOG_TU{xRly&2}kr2&M$$qe(5ZFDfJG4Ap8v8Xk&D&xqMJ3$KPP zGK+o1dE!(Q0Z8}Mng^4ZWlx9d4YxPF?Zk0w>fPXtPfSu5RiWk~ZOVa+bxQv2-WIwr z;XlE(7PwUg!NNk@ElTGXa`asUJ}#AGnrYN6eX?k{92;5UNK7E(Ep|>Ex(Q}^Q>Nms zK9y_2M^zHy-rdAjPtJZ3SuDXWS69{_i~ci>&o?eVtSZi*Sr6HJX6$OI7{aA9BYeJ1 zG?K?G(u8g0>81TAeC9dkNR8vINEX{)tP186(CD#G{`ut-Akm3-sMDT^NPdvc#v}0^ z0xBN4UTa!fZiCG;RxpA`doo@XBCv7G-TMIslXS3h!z+{eM1(tKzWi1zk$&bYuhxg; z>yMjswd24V^EiHJh`zZ1m}2kiG2sM8c3i;PUUN4=NJFJRcC(2kfJLqOKAI6TL%+jR zTMF23UO<-4A<6|zwjk-pW3*n*AmhzGqa6;uwK1oX(tBZVnljN#xL&VY^4tv=8Y?Kk zZDcOGWaqgTRL-3k7(z&mkDGDR%te$mZs!dH&p-HE4xvcTyq9SF+vG9^44ed0Bc}29 zYW6LX3v!~vLJ)!jz=VAsQZ0QO?aBJEI)T(CFo^72AWhg0U$%s4;SI#kcbDZ0ug+-+ zFz#zT6Vqem^0v%g9_Ntx)Q1lbZH@)om!JeE=w}2DPZ)$>=lLq|$<;YrMe5zQSv54B`X=;#fsldI+~$BL!Leb`2b-hOPA zxh7Jy0v*2TC)3wJ0Oi!m74@H_&F~5`{ZwO)X3bHSaYJ*71t~{f+*;g`9ROH++42f-0AW|LF@3rwEo^0a0_p}r*?!O z{vxIRBM17#-Etl@G%mw7%#H>-)}q`9fN>vs#@au6JcbI(FIC`}MYjf63fx9|)(=YC zqjbeMVCWFI%(LK+#Sf#NX)rPD{Yx{`z#t3%*DiEPBUGDB4u@{GHZ4CKx0+Rqv}It? zYPWlmD)+6KimR(H{=yD0P1)-GM3rk|aFshSz@hhsQF}ICSK886aBsMm2%tx>E6O-Y zOM<1SmHWG%U035}3?B|35zDWL#jWEBeBUDNfiv4zsJ43NwQ3rSB1N^I< zFu9IG9hyS_pmsBX+Yt$9vDuLQ**&z$hlqBz)Lq$Q=s8yQjv+Tcy9G<=xE*Gr9&JDM zN0N2N6xr2JhddA^zgB+_s`4vo4c+q?9+}#Vx+cww&%%0&N#;pXIsA+QOk7=QFH=Hq z%1#_QR<{Kr#CCelzhow$-W<NoW`8Mr`H=B+bpNW;1fRz#Dq$u2l9blHNCY7J6XS5MB zq!Nyj$e{dWu`Y@aJIoL&MbS+VO?Iy6npw|>TPHg#q_69?H7>%)|5&NdboHtC5E*jK z8ce6~rg8{p372Zv=nUE8AUmvKSH#qaUa1M|%d*-Vni8W9j2F)@ms2IGz7$njAsCnQ zOVr=r>mwm7EOpDb-NsZ5QG9gfI0^l(A@)q&PnD?mE;f`@i_h9{dh4T6pFD$Zrxm?F zHiroObgM8Fx*5G0CCp6Ts=YWjZ zIP%iZ>`D90H?s<^G77>!luFrQEh|wssK<<^i{tW7z!2l8GprcjC&5{bhezu3zu`? zQXFI@$MCa0m-G1S#ZU{|Z{f$kFd}JRyX8n4uGFEX*mtLqP*XYu4Ia{6gAWHvE-&e% zs>=K5JB&cH(N!Wu0G`h{ic8H_2&0!YlAQ6VZDS#j4ncy|<7yrEsop}I`>N!O&J%v| zMb0Iv57oX_9nz}KpVk2raV_a|jgtO^{xlVwc2d%g{_%7z`XyAW189%vzm8fHPJZ*ovI(#X{UQWJe*pZFMMe! zzu_z1=2fwLy1X0WG`9Z8xdyiWP$LU3bo${Oxt*WKMep-Xe7po7{(7yyIlxUR@m%Y+ zs5`WSd`*?#jATIFPL&BilRUz0cLf{Jdfh5qInTr%N*g#&3gE84nqrs-5srHcXEY}G zx&{z9cptf1njTAh6fmCI{Y?-^D9Jp3q*a2C3u@8dg8*gV7TuJ~?FIJhG4PPD#sXr0 zRlW8-;?|=pLJmTp`oH+r+$7*tZjBrn9`r?d$=zo7WiR(fn5QL-#p+Q$t&Wy7)^(r# zdX&qmU^r!4Ar*lfvqi1^gI{d8iK8mFIEawMml@zy0ame1JCq|7f762%n6Z(sv%xL*e?(pLaD6PCdc8mB8cg1~C_7_XD1Dx`! zWbokrXR^bPutjjiJ6g&q-d%T7?S#YZcml$*%eEmz%+?_IISgDr)G`*4px@5m@l(3g zRd{|S>HNI`D-3x|wJNh~ku}(jTJdX5^gb7U_jz#>*RAB@6MF;;ff8aUdjEC;oY%Y+ zF+1ZQPG>fc8^G*Bao$^RZebO+b4*rMbpH3nSi7(y#9V)VX!gd49t#-eyVu%hao2X- zMECsi@Hy*z(d3x^Oy%Kj03M3?eeVlgiAy+^Bn5LMOP@309`?7cB`*oB4}URk$2NVi zX|deaRFkRvy3bXg?u~wSTEcF}T>s@ZSnU?kXEI1VwU!n?#HT&tr)x?Jx|AZS;sF#S z=m-;>bOLIrA;9&gr73v8IP0j?$!3~lOR#W=XZ&25|^55ow zEWT4=&EcK1AP=bkLs^Ww`L%7a=QEH+N8kM>=Z;B@Xx2?kSK~y2sj02)E-N%hl36(i zVfK+)apu9ML>J)uMVH5$oH|Mb_4srcK`Qy{=IZHij|snp!p=BUp8M)M!DVnx!kY3) zW>VvLeBgFo9W6~LfE%TauLxhKB_$BhYHQ_-Ea`f}j@O>lAa_az;n6G$7)JSr!sDtijVJWab=aVYcw4V--VFPH&2M1nC zY|dML#h(V*NTgC+@s@u@i~UWJ%KQxUqanm}$o_;=85sADT@e(!Bx@-^{2aOWdi%m%)JKMcSWw55b@VX zyDGz2sCXPdy9h%8U;Y2P@MMYvYZ+07xy@HeD(ek%5;x>?+rhuly%XrH%z4AQS1w}` z?COJ6mqih)pQ6c`737pTb*{koHvex*Uao#Y|7iZjT>UDk2lbNl{hI}zS$JcRBrEk& z4tW}9!y)gmTAez!T{8xWb&aZ(+GC}{IY9MYPIrx8tG7*2sE>Mp`4Ea$yS#6f`ByOp z!YAKSZ^s%>p-_WH6PeD2$q8n!_|*3Cqtl`u90nGd`Dv{vkVB^Br02;RZa4JZ;1?zy$qFlYt|P_VeLU+=tv`H_Kk$cDngoMV`H05Y8%65 zCRbbH#k3^z>(STb+TqxEtOgl8oEeEKvWrr=c06(ImH5=zny1x#puj%xtv`>PN2CvI zE9R*Ob=&wKTLY7<@*4_-TO@zBHn8TfHJ(KN9QVU;Z^AEfVfD`0-3NgdawqNnU*MS! z7NueO=bqgV#EKY$G$`wJ38D7;h05+Vi4KSh0xZ(T_Pxs$s}q=p{%%TCD07lJMHKF+{Yh*fPjAyL?YzmEx9ZQ*|^ zJnN*WK1^gJtPB-r`2lOvHd_wKrK^X|iDpp{IeJuhPkUK17f%K;D{fE1ysUf+WP0+ ze;Wg=fc0jPq$*vZU00(j;`Fyb$Q02$#I@H>9nSyn9Ui;!11#}t#@xM!QIsZinfR4KnP|-VS4dSuo_+H4~9dS?xXS*;6V<+o0P8xn`_THy~ z1P31H?h1PAfx`3{h)4=*=>>?jNkONi$AgH@e6U*_VV8y3^>CTj%coJspx*zsJb^*P zG7Dp>TzRQ(q%aTBkTawH#)wceR|<#^*u2 z=x23qAUVH(?=E>ie$qKFDdHqVvgLlJqRKMY|sZmu?$#|4gh4hXAJr5!9uP1$M8HgkCy~U@W)P%NxT^Gz; z)L}OnJSvc*?$Vnd0Hw=Pd#Mg9${Om1@M9S|!%>PN;3<6O7B`lyT6`%?(v<*BHnmT} zS8>2PA)k^M>`GF2d!H4XGEFSckSDUkQz5(4}qSK zIV`?&+Vn^B;QY;D*ajmlJB#n&ww+DS{$)6(%Q78CKJ@0h6%gUyq#a0{Ir1d$0tDJQ zo_S4kab@sR#n)6Sn8XIs3|NlcOXQe+Be5Jy8Rk^Lnr6N5YWmBay2irnAiqV}O_7zE ztyPl^&^}mdSw2HMFDt=ToTI>68fP;+B4JsDw#1#GFLQ#umH`MqjrPE^FW=593_Yf| z32{4;&`tpqi|}vMl~d$lkF=k@WozH#B;~Z2<^GhMWTPn%`TWwF7u??G{?ddPOSue{ z`2wZ~o_J8n{q_@ry#-aPQ9|bPt=epM!O9wdgSIlI8@eyj0`YN`SlCl zDlT?MZq!~UTP5H+{Q1+kw?^@jWJFFT1(vDFvgd^_r%nq(+|(O_UgkIM(hwFWhuUSG zL?7m4=ow@9yfXg^>mF9bc95#5p-<%~nXAK#pp&fCVZjchylk2-in9~fCDioluPv?F zJhL8TXijbSO}4Q%r_vH4;oU?e)&IO?E03$jo~npI;q$E1Q&^~)5Z2%?`!$*dTqb<& zpYbsf>=}teC7$aqH#WPBH{$EaxtkuyN&+TGrfcRfTdG=yXawgUF4rr6^ei~Q_ht;yb2Cl^S)|TsYhLB8UyZqz0-U zZOU5`i5arXa$W=Vz)AA4`e*5X@_nCwjMYD3AJT!i?A;u0mr?D_wTADNr6KBsT>D=U z3{^aKFT5|Co$9c=i4`k63{*WBLUQEAT*s+5qDCPIK>0iVRlEQbY?-UEo6Qmm+;>h4 zr6?oNq6T%((Q7?os&gUte#WG0Hq)9v64YC6p8R>RlPqhj{5w^Ah3vbDt^e9Ovj03& z^L@|BF{h{!8nXYDAbI%aTz?fe51F#1$vVf1A`q!eNyCW4$MY*`-e!C|cbRYgqQR>G zivODwU%x;WG5fag_x>z>H{p7feqBJ!rb3}u3T~2B6+PMmi-T2SS2AOGcS{ z4VBNGqT`)aVdIVtvKscly?()1_$;qX@>eINBsAvpC42B@=C1#4c~8Lse6wO>Wl?dr zPwcoES6{Xn|Ifp!$c*o}SS+0hmku8M68l5v_!aZXz209pWySVn{YWNW8&A9n_RXgW z{gT?B4K$*zPGI&wW9)e}df(k^bvkWInx-h%SO^*5z2#%%wzYglWXW%Dk8=n|VV$k$#@;Ix9}O&6O{D0{`2c zdjKo678oTj$H8GD*#uXFmQyl+=;!{V)+FcxS=GK>0v&qiei?KUv#+goSC6Y3ZC2DS zs+ib6yIgTJ*z!oDSP9JE+jAUJILv9ctA_3Re4Erj(pcb%7|c8nt-u5XmMARh4Cp*C ze5-`W&GSD@s^Hc4u`%Eu1a9+m1-J8>CzL$d~EA;y+@L~oAQ+qG!j01 z$EA_8*Br!1O?PRMtZn&h25KMEFIOQgay5qP^h$!i-zF$1A8x<~1j24r2*ZB7LfR~@ zi6xRTW6lVr`2}SdMpY;{Qi68ze0Z2Aq22J3hI7%Xoa1fn02Y7hb~HLqBaQljAwOY1e$m0@k2xUJJ4Y-4;h%IB?Mzac{jIT&9d#A5rrIjY~e6VQhB%~=U zu6}X*_ZyNtq_QYuH3MN{pd{?gyyB>Efzq$|96SDtSSQz)bSmlva;11J!E*WV4apL}%K!r>`sZrQyHo)e#s6<`(BJbDG$C(ch|JaXCO!sv7Fh z7kws`4yb>OeDN7!hMwn*#@5gtSziC3_fzlGJy6;+)mnLU@0j=>1lrivb=>AYEyuUE?y6ew#@F<+<4mMCxqOp+io;pS0K5DU~=T+q>^C0gL{)v#bXXmQktDDb^`AnO z??DUIf?Rt zDCei=W#8w#r~5gsiOhxKtMc5F$k&}5Ljky_28hQG(pZ7eJQv7`=er3tL)Y4}1z2sR z77;h<`XgksFqyR`{A_N23?hI`$Edj}=?PH_xIFl`gnAV?WOPvKEE)vIG-1^5#s99| z>>6NcS>zl#cmAjE+JNS>p|y&caE7tW!;$bY1A(1#=&;1PV`CYog=UdIHd?ckEHdqn zuQt?hMIVIsZKcpMzy;k&IZ29cR-&|8`m5A^ugj3}ueQCtPy!I--wPFFaxpp=(szzz zj(f7MOAbbKR?$+L7gq6Ow@}tWYcvPY-QRs$F6><6Jm8oMhfj9^7M3m2%-(5c#$yw} zcQMBCkPN1@h(vYTX=fqciM~y>^Ichfg@_HOWyWE(RZ5C>M~60buLUu=y665qUbbQ^ zb)|EWh7Px^&#%>vkZ&f!~_5UIlN=$(Uj8Y;XLKVM)i1r{;3ogc{k29Tgd1!15IcPUzB zPb3Z4B)eMad<>i0m*Oeys#<)&Pgg(q_g#EzsN6Nf$Mi1Keg!t(cN|hvG`5NgvZ@M; z(90~*Wl*InRr@!k)4ZNKCRtbdi31b-gjc`jDk~-o0il)xRe^;+7jML{*C3Al(2?ge zKC(mY76q951rL29Lw3 z6~P*w@mwP@xg(pB54fKI-}N{D+Xn)tf}60ElfN;Jn0H`=Sp}J*H85ntKaYis(jF5R z9_?U-omwsMc@(lnHPZIV>RNhEu=sOq%}mFuK+3JgO0sxkaTGq8mar*`^_@4_p0@zg z=Ps+mgE+aKDNFlo(Ld-PO9tXHXgT;2+4z(gl5^4R~!CLi_bMawY?&SKkV${W}59ucVLoV-1ue- zj1$A*C4Gr&2rs&M%q48|sHzKUop?a0%pppgN|%PtzCfB8k8N0;6>`HT$lY}z^5JOi z#Sb&s5BopNg}+eCD@|Cb2sY>QR1m6mvW|%nHWwE@DLyR{Egkb`oWWjt&hs_f;h@lF ziw()F+MN|k``|Kxnh(DcQZBQg?XA5_0}cMXr!z)awB|DNMWml$XNL1|G@#Teb9&(U zZ8!xwe)e~@X7NwJ=%m*}gg4}q&AQ&bM{}yo*BB_R*gk0v&S z{!as33oL3;)rj?p`}eIL&*Cfo-_@E(e1m5fjAdDT>oV;6|4WDI|N5+nP>eQtkMy~h zi*I$ewrU{;V~FeC^BRQzB|LnvqJixL%U}^X$x+~j<prsL!gH`jK)95_ZAA5S>!V#T;(jfUhlzB>h}&rIK&6ol#@))f(E&{@P- zcwuY*u@}J%|4dU&ZIR0OC23QxM{I%uT<^{f?}5kmhm8r-r*E=zBDhjRJ)DAG8ll`= zkuJeT3#W$2=g8xnavII$*M&kB&H8SBg1Z!l%rhK727HJqkAPs6LHPwqv=%Dp*(92_bQff{}`OZise2ld> zKs@1)EPX&o(Y~e5LcqUom-fo~;`nosUymByg}u5^uzvfqSwg=FcuU9H_o+_lvKt&> zmPNO-ve9-|rG<*;>y%BxUm?}{%At&(C}^ij z)JB9Cj{yq5J>Kk3mSk`{advL?f7|P~xva2;_FEqrhTw8ptj%8SnQb`igWJB=2Aqji z6i`P8w3k?%pW^aR9R`fd*8B9$HQoq?8)a{izF+dtV{(4NgtZ)a#ae%9jZXSO81EL4 za+JVJx+jVPhbGTuzBHc$@8?xA2Guc0rtkk7=m4Vu-S`Fc{>`-b% zbPt`0tE26spq%mQlgQkQ^HDga0h95HT^H>Rgl zM74IF0(zHo@N%u`)05% zZ>ZTW!;MR(*`g5H!GhH#+UoHsWDN7Y+Tt=cH=SMe?@mZ&)URbKRZ|e)Oy3x$vB2Dn z{o@L+MIGjrgZ@{=QIaJ+Pti1su_~k{?ImtPwR9oeby?D-QIOeu*xFaT9=j6XF9`Z< zXib?x7)h`$Rk`;a^z%AbtBUc&J}oTZR^8M+okhStpiT#` zP-dFi;=LUksa4-E&rvrJ$Ro<$DJg;LEW`_W`fZwF)9?u=>xdwI%(a^+;M&_l5oPh|LJA?&>oBI zwnI#@WfEjb#18k-HDiX?q#q|qAOU&oeAMhkQwh}#OymVIJ6=~XOesrXlV|N_(ZXlc zBJxgl`>T{Bduyxmr+IP&-RW%Os8L!|eAiiHytN$vXs-=lIg-ZEVecd9pvm%avsqC^ zVs-~_O^@K#To6TaNm{m&PmkBEh4&t4+`IpB!?^0a&9s!>z^;#3l(qaaG3h|T@J}0OB&Zl5i28T9avpC7 z(MCV`{Sy;ARqduQO#xi0+YET+Y+WhZNY}=XTw7f%T*x~)r&J-6!df3`LQwPF>8-ol z8I3F!IIT6Rbh%Va*>EHBm(Dn9}oF!7#|9+IB?~4x3q4 zPZV`XOTzjk>E(nqVSoG6cKI4?J`BRfTV?wfPB78pVdK5490Gy#qNJ%nFgYzy$0+Pr z&oj)yIHPe|ZZoISeAg!#sp*)(CiZ;2GZGqm^DCK;0@_~bx%kGMpq zr0XnF&-iZYGD&;*c|KQVZu?4@wNqUJpg3>+8{=-s>(ky;WFq_*Jr(12gk`=;`%xATlr>8NrOFglDgJ-=+V4nTIUG=&ojt@svj)>D`9J%1mU z%r5Pk%m!(fu1$-mijs9NBg6J?Z)QqZ%UQ;c@wk8I?yRudXV3z>gMvAmDaOC z=x6awCD>tG%5d{1f+?n52cb?^1WtXlX*7v9=JJ)(g;{yzmxZ&n(+)pf(H64ImwY)M z2Dcp_m8RG|oZ0Oe5T45arL%U2E=tT+;VOAB8Sy2jP&xRjR+P1pC^A>8acfKTJ}p0d zS*K&4$5s?lvwsocre5PTC&mwzkf8yYo&V$wSYa<0Eps}2T2?`Y9KgCjELDMEU_~5% zWnuM+`N}PYHYh+sXv8re1?qJ^6@T_s$8W{7g?gvRR)iT|o18&}Y{ahLa__3o#r3pM zr>dLfy|{-q#BHv@CEncft6N!)KIH#C@y_)wlM8<`)5+tB0=%dh0+hl8Gh!Da$C%Zf$of2qg^&eZTsE3V!%F9wq-PY|)IX6dD z95Ug0B2C&9%aJW1YC~lUB=PRYzosvG^9e%mQFRhlJr_;Q3Uw4dZD%F|yS?uun5cS6 z?j_R2%C+|FN9q*LgmL5RYq+~7k8&&Z|&!tV5&VWZI zvI3V2*FU08SEz2bczlG}&!r#if*f=j*yb&nmu>K{QYj=#SnR?p^qbY*{N_GW@p6dz z4;_Zbc;irTMV|qb=7emQzCT89L4|nTz$i)lcO#SMANKK6EmJFt`RB{`Mn`mqFO=#R zt#lpBR0Ok6W)1la#XQ|AC=rq;n9C@UC-v4IWgq%j>wgeB8b}aKcu||sw|Gpn(>WC3 znCN1aQWGfRFq}nbsBRKfP;3U#rSg_dDQxXj&&TKm%$WB>1tsRub_uVydi}-LBxo8O zcS^o>zYWQZXz~`U#i#MOCr*vzGT7Qw#pUhc9|iTG4-Y1HTMxI(kEhGTV;z?>hAF$DqP0U8v<6Ot{ttIp9-zx&!mFpFa=F1@xkQjyjOL3*LQ!DX;kQ2rIQew#z$hKq}B zo5NI%rQcSW*d2%!(a=7v3!}Jz8=!*Cb@Tp0mCc5KG6zyp6!+ygA)R^TAKpTRznc9G zuYXbxL69a(a>O*I&Ri`+fTxu0f4bYqFZCB3+$#Fo$SU!Ucor2%qL+m~qbYP+ z8`t`@Bt8CN`1Q9LD*r|O59)jnf6+R#*5~wtQEuH3U`!veQJiF4O&?$5hHS8p$2j*8 z9ke6*Em%aJu#s@?s2rD+e}7E2blpgOS-TlvFyl5+V!q6_xI%UMysP|EshG>KdQ7Dv z1p#!_k82NO@{R;8?r#THbNZVJ?xzC)B|Xt->kMx@-II7dr5HBQ-0ahEqS?IEwa)O6 z^GC`W(c1%|5CG+jha~|p$4t+(3fE@x_H@QV+ugLvBzF-uY~^%fNI3;SVPy-ak#m*@ zMV9L(YTeS?(n}*qtvKxEJC38-Tw@KVN|#`?)JnGdbPfzCh&F=XY&`m_?9n{w1*%Ha zSc0ec`8)dRki2Fsu@Z*)h2QNbpKtln@w!~;+pk#$gj)hX7W#idN$~fG&$;Ii=egNx zHN{hAiD1~Q9}Q7aPC@%}`G*P_;2Ng6pt3e!Q+~P{E2q*B*!3zCeOoE-jU z+FOmcp8t)kWjlO#=uRnd{knM6VGvJE+n8{kM?N(Bp(D(A4Ia69V<4(=U+>-EKK zKx3o9M}6h=AMR$7{tO?JTV3?*Bz^QM;VEIOlK;GJ@Zr9Vsn_TzCjcrljbY=# zz>d%xqer^Dcq+#;Mrag zt~O9VES!WTBkZP=a;hm_*P)r8M82b25O?IV7dG-{r{S%6m0L)&C+k)DhUuMZ*x^Ej ze1+IZR*OrgPy1?;b8F5=oG*IlQ@ddA8_Mw;A@c*defl+8>ZF!;1e{ae0g|w=<=DzB24R#@wSf#Qlixiau@Q+L zfx}vDOd7VjypB3;IIJ8CzHZasqWS?tjZRO@)oWb1*hU-a`IHA|Oe+^IB`tW%nnj3( zZl4QDOh!>oRVBO$G!;||*&(KsPcHf>{w4zC>r>zkXRXh%Yi%=F#z=7CFFU);@d;); z&`pfluS;`D$Fne{s}aSTcGH$ozaFnP7rf`mdP)+n26{R1*H`hSS*R`K>rW{Q-0(zO z@xFCEN(7!rrEpd*F|kgK+C*+*ViW%UNS~t^xX`Cs6n%zpyhW3zzRayWDI$m@4>C}- zo^GnHl>Xv0>?q{-SDdC-`hjWa#>^5#Jk#pJX7`*Bi;@Gtl>iSbXeGZ~o6p+g%!q;4 zy2|wX$KeT?7nvp6)C02aM?neiRNSdz0 zrx#RT7(iiJsBdNXPJg7Tw&DemJRNLZu{AvQsX<%sT-5Xe0~v>srP---`Yq$Hmwo@fr zz_w;Wx6m6?Y%Bs)a|BgG5LJz+rG%^c<)ShQtVb6Rm0L2J8o-ag&Ak+oXEztM8G0DI ze90nxxhqc(BOVW^))K$36=DCSTy$x^6U?^`Oh+wknNy zo9-X=v3M8_$8a$^E@;rJ;bg zGSqmMb%8z)-;-Y&ETD;h-+euZ>S6@g<7r8`X$DMau8E@%AU zF~u|Y%;slGZO$fYARm%7=T%fp_UAR5pI51`16j^%I?>{7I`UXIL}%EoA=olyt?p$- z(oyP^)W6rVnNc;oRz~8I>AgEogl5HP$N%%ykyx+#fF4SO?#m_bV}*t>v;so1ZhWc` zr=uUgKZIX*TJ}JF6(9(^**aC5F%04=`H(GP&Kh9q=6;xOi!U-7j01mWs6ttJpRgXl0RYNsGK`=Q&g`(UAUCYIHg61Ue2lavtylvE6; zpqXWW=hyzVRtBvWxzlcRMM4Eg0&pvNau8w2_1kMjgzx?LOacm)F{2Q!$@EY4@~N5o z3siGMfR`bko-G|rM|^1_F`c8=`)CAw6wP8Ee7BACXcc6k9&FBtW_&yDKTT#$J0)Ub z`({HSrkhLc$=lln5?z%Nii3l^pQNw*etPK1N~^1lS!+wT2`knsbVCQLR7c&{SMG2| z2vZ8)JmIO%cygApCr!IOJhxYMiDqpq$ta279gM?r_Ht<{I8;)m#Z`0AupJk@@+UXB zV@sBL{!dxDO>4tN?^87T7t)L%h?GMQ0Yw=?f9mDqo~u;dWm;y#=Q(04Bw=6pLRjSl zqC5$I$NavMc+Dzz&Rpy8iD0O`VnY|ec#uq0fuW%rfBe$xOT|Fd>*mA0NFUPPyjS#? z$8V$MrNv*YsS$ynes|PpomfjEuc-AdV3^sY6VYDpVYBUEmRlCaTnl?8iL(~=KG3J& z9P)BW?->1sX}8v_Ws)7jylwi#*#9-XJAip(-Xub*Ysbq1%j=P%)B47QnjTs&FAOif zd4BsI=zfzM6zp-e%SMaQ#P#6Sro5m@2(0fz@y*f2$&hRnSo(V&be=jdBpJ@dWjPBq zQ&g7h3l{^;ltttz1jxWqB}!5xIM%sVLKmY4NRn>Xw1v_pcM5SY~RiaGd?h8ya{$_d4Ob*X&R5`l!YCP&Fc8s43~IcEjHpH4An6 zl)A6eiJpsojQ=iby=_qBBbo6Hy*mO^0PUs3{?++Oo*qOsVx7uFt4CU1DgM)7t*3nN z>VnD>mC@Ae0ucdPU}PYzvneC+wU<|m&S25`*4He}8*9x*`+;=CsEJHnxMg%c?vUnC zguy~P4l(IuJ-=>OgSk`K{GdedT@3NdmjhN3tgFD*OJk93rvF=eH8S<0QW&ir zL44{HH1Q5f@^L>^RZ|=@by(4#3#kI>v~xVyNvW|+k%MO znP-GWH*qe96=kW|Zk-useduRn&WE8O?{VWd?4IrULtiS%`c1Fx%W+b1eaw5{exRC` zP73Qut{gH}4;OQFop@sTWjYg-tWSI}>hl!MJr&Gn5raNqGhu!78K+U7Xl@Y>v!JL@f4r!&bS%s|weV<0tLSN2+hva1xu2pp%h z^nBJ)dQHzId5V>4ynyiGufrHO5pKH2W#yP(vfuUVO814_*^jUyBtwSh zBcsk29qua;ukQ>Ww$2#`;Ml!vknSjbMC}dQw^wvXB6cqRGR1|{3?PnQarDAI4Ep!` zfi^8fJkO4Q$XfnG$KB)p%;Wied{9$-Q1W&~$Nj}<#SPM=J^K*R{@8Bvc#{5b(_nJ8 zdpTngbbDHHbMDe{mq*g^uz$OuvfLqF9dvh-{djGHF71}O-${AAK-YS6Ke#{MA3k0m z;%qP9meRjJwdXW=1PK*a`6AOeyvzeW0i)&v-+&pi<$KrGA9wef zexreZcl-9jI_Km=)GhTg5tZI>U6>@UyFZS}JOpI>(e>psjQj_`NK4Xg;y&uQS7|>( zxoI&DmU-m_da$iWRc*V*(Z+*qG5Rtv6}+=jXSVnOe;uOU|BHpKj*9(vT`<-~4p{k? z6%GyTO~Tgkf4b8CLK$kG+jTI>TgaiTm%)2^oJ0Yl*Dr+V4*cnK<#FZk?wTL?ZgipG zd`eP;Ps;N$SL=liB^O7)E*^kXat}USVrM3REw}MY4-F8ua-$2i_{%~ag0IERhA{Ih zJ)Pcw#=hrH09jN7*TE6}=3eyI^)kv2ob1M!9ea zc>y@=rE2A7e}~O|g$ds}GEn`imMNPKofFdfVzz}YRy*4-vNRt-s{BSbZDtl|5|v;* z`=a&djlH`xPck<)MwRN@CXF3+#YiIBNK;cSvxIs2U)2He!msC#d%V(`LVkN?gm>F! z#8OpwWaE7J4ZR6Q&)+R5K?6Mk{maD9_+`eO1G4Ih7K@_CTm(Ewxba0J-I*L6b|4R*|8cy+?M9o zdNWR--eD1|9WqJwUAreeDmgS*iaJS_gu)PaxNaYU6XLWK)Gbq8=S6_koC4`s!K`w| zyWOY|zv@P<-hPPb@l$fgD#726KXNdJG};f6^L2s!xUUTq@%2%nSRPG~Atam9%r`s0 zO*$Pm1=7;gKTO`xk9hk5&R>kl!YV|F;735Ee>Qq*u>sDkr`jdy@yLo!b>lu1?3w`q zcj%UiKlx8uWvz5(C3=OEQO?sL>n|J#!+t%N*SbBhsaWXw*q( zBj|CYDxceb9Z<#@SO_cok;h>Qr2B?;XApO3LGRx(fn>IP?IR zyN~M@nATBHU8dJbk`z$;NgWVWmY-Gm_VDM}_7TBJrd$qQ#a;X=SZK8K-9Nrh$N;fM zN?}#$NT=-nTUQcn9+ky^5xGoXXN;sP80{mm`i}+t{si2qfU0n$wuC( zkxD`6!EwygD&RKpMF#nY?BIrE3X*>aC=8esS6~eA{$lK3x@YhYD(JWyIOx{s1!bT) zW|qCAr6{M>Y?Y;Vztsf$2mF0Oc!R z)QL4O0Sj`IPmXI>I>IRjZ(Mm(r-~tkmW6U~%ax9D-G+ejA$)I228qVvh_IN<$MiMA z6W&Df6cr=8q=wxBc$N-bo6|3&yO4p!rGGcs$*M20IhbDcAHuhG1J0Q7B{LlNB(IY* z^V71l-lG93^7+%h(CW(L`Jg@d){X!Q=)bQ{qzaOMhXBIja3w|D4}7H&@h*-NRCtuI z%^j(cAsRW5elMl<`<7%j$(mt!XF`t@Tl06*gYT>X_K7lacWGIp>6mK-Q$J2v+<%7% znD_vXVGR9@N3>4nw&z^z>r(UxK zP#YF4!rW$b3xhn_=O-lP9~s@TBBSL{VPz5_mk*I0lN=BWP_vgz;1#_k3y%ff?qo^+ zDNi(1wbbDnL(DuLe~r;ae|kKbTIcIbn6VMKqyts6#!Vp1y)0lkxn@v@b21|n7))$`E@6i*9ku?;IYB_F6ylx&EGHO+Is zr)~OX8a8XAIA;9g(_%F(E^Vp*tnhky+q4p2I={W~fZOb#0t{u^D8m%yn0U7I?~>f; za5n%;OWDK*&=>dNc_ajx^ z&sda^Y-VM%*$^3$E6knRJs{5)~yh;IZ&HZOZpHGUrMEMh!jMf&J~keAjf_{5WdyZ~!<1 zBqN!naK4JOke7f`Di4Fwr!X>HW*ye7h4&Zi->6mG@8i8wksgRlTGeB2{7ZJTTImy| zKK^ve#L3=tQxGo`s&C=Y56C$Ytgox<9E~-Fvsa1dCu7vF>$nPc79-fo-hGnH!8KZN zL0OWYjdIBO@ji1n*@T_)Dt4t{Co>IUAeW&b@^*@SEwE zuI(h%%Px*x3cq|wPd@_QigPY1@75aZq5@Hv%g)B461X-ccBY~jY=0}tcs1VmH0c`# zXLQQrO^o^2TPq{bbBv&Cg})|o9jWu@R4d+`vEyfR=*uz168BLzXBO-eo2f3lq2*iDfY88OYb>Z^*h>HHcj|fV)7zzx|%s%nX3!L+ggs_IkY6F&Dh?{ z`P^dNx`Zz;6Lq7KWsebyly1pM0$k-6mXKZ`oB*_?L6kys8-KdZ+i0Ujo72%?U0riV zD&Vgvax&I=%5i>=e696%>7t%O6AlfPW<<#%Syr zNob`kycUA?qo!`$yvkO9k`FM2Iu~M0sw*BuZ4-H5s3zh0n>`guvt&xT%Z;XxDMOH# z5);L6I^c224$z($}tiA0rn zPFog^+`I@juL_ULAz>gz7}ukU*Ekp#(+NXji?~Dmfq%v@-~bhYvijATjr{b@UeSOJ z;_b*=*pSsTEz7SVY+5~2;DYWGdC}+vK?U_p5mHdm2EFwI;RxBButkT}EEdRmm7I!> zPMM>ePuI>#rwGWOFPG-U^+ydJQ^_gJaTKM<%O)AOfqyWRFJ-c{-(>C>c;rAGkG5kE zyY2I0>rtjC@%frL(X-C|-`yn!NrEos)@ux8Wf;oCj~)jKfr(FM9#^Gtc@fNVp#(6x_riw*vuufg@3(kn9fA5mVWzjbwx)-Y-OZu6dj%Mk!c}WFAqfGV&=w?}h zUeGvhNfey9dvXZZO8g6#KA1DXvBRhbu?BLR*KRo(a8*ZvAh%EOA8IwnLo!lpCpY;8aR77|{9 zQBYOdTKp!)>APFLttA=2|44k$JqRRRjpYy@wYYVg$8kLWvFt_h zqRi%YC`&)>ZQuXEm1%QIv5&NLBs#9Ry$!kcNxBNF1M)I25TltGLVn@MvE3VDr z(XL?26`S^DIAJW4699N~_-ViiqdB;P_mHXf zUVE!v&LtGOJhB-(QxAJCC0eEbPHMDM{~a^vE^p$Ov2RpEf^z}l?@&!+U}0BbXWgMF zwN?coRD34kI)dW7I3*^H0NUtLl^AZLp4zjc;0xb9V7Pe)O2vQD%54y3M9Ee8c&XaouCIP#S!agu6A5z^`~Xvl4f4hP~^!V?jNhb#jJDjAYIM0+^==4$6QLj{7@>aBcS$NfX-wd4A6Hwtc10?rOzRqpA=Y%+o!3}no3Z*L^ zd7zyJEp9(TT;u^OWjA#(tTB6Q~lggkBw$m|rC1;fey^*h)3OiO|Em^X>-tp2r2OIqW{kX7&2pC&RyN zkR&g>ll7LHrcsW68eRoGy5h}Y-duc#@Ps&R1-)g5TXY#gIgGYYzH+2H@VohZW(t*8 z1ECiFj}Z3W3%|pH%kz-$PUn%19}P!*f3T9bo^nduvGyfdi*{rH+*L3)O5e%_bRt!G zg9Dyi&v|e|{$(z~NddT2toI$|ZUP0c?Ps-DQb&<9{-7Pv^Pe&dilgeOVnnRoPY8oz zpPYIxTxt1`J~cHctZxPTRFE0EG=`#jP-$+R4pc#}kmLf{Ld+R~C#Af-vB~_oZ-dRqAY|$tUo&_7Okgdg|E5nz79_$@dfS^kUUkiWZq6@LcveynGkl`ni0P`>>- z1jZ?yY|Hb{gu~U2huE!Tpl{<{eJvLkD_p zX5UwRaT*egC23&K$hr@v)FDR|``4B^OuUsGV($+LZ4{mC;vyKUMA}tolqYv`p?q1I zRSSAiZ274jHTm~rxgFNf$_7*qY1pb-rn|Yo>93Z+%0*O~&2O=(T+26D5Rww_7Vq={ z)W)nk7Qzt9m{rcg{Vha4DJVn(B~S|~h&rSf4dNRih=L8SpGG^`5y}{MliaNPT(#@8 z51i8Fw@meA#m-)ZB#P`Umq zwEH?{%Y?y_gV5fo+_ud+S;Ru|kv(H+GLJaT!ng9Mf()^;Ic@fZ7n>95WTp_Ii;quh z*tEPcT8(`VjnTK2hiZs{HAiC5~5~e<6BM?NVFf)7eqBb zz{YHVFiB@2^puY2;1(+Ynd*IA}qT}lZe7X|X<0}%Eiy!6a^E$^rZ_ zP!^a&NTN&Ax6WmTQdf}P?<0)zsfCqfmrIWz=)*O)`a4=rDm!GJNJvKw3M+M~PlzZ$ zN(OOU=GODt#n&`~$EgFyjZ&+UhY&D_@OXB{ zT8_Hf+)9Z3!)i`4|8rs~4w&sIEmmLL|%6X9Y2Gx2|MZn?R%%(5TTAmA>=@9|H{-A z=Qa-+_tF#sWvg4M8${6Ztgznb3GM^kMj~?emX&Cgr|K2Iu%7-$sMQgGA(~?tUW+Tp zay$+jxrhmmvlpLM-FtH?gbR3f=Khy#h~JFoi_(j(!w2e{<3OmIY+bLN96o5>&G^8<(CqvZNS?)CYQpmXsavY6FEKkOiRP$~S zG%KB5&f8dGM-wPmGt(Dtk@5ZNmCj095J#xo*QdZoX+KL#Oz)QA=Bk-M!+0E>K52@> z+<9q#!J>C!P<^TX4hPg3nc|MEsgI*0b91oeQvZ}S_4lXi_6sdSp>>p{Ups9*+GQ6S z4jr}PEq{_VNVFZ)%FqvSnS07J>p|mEY9MO7j|Tn9xngBdAKqog0Grc0 z8-PYWsjGpk47a`hqfGA&Y?SQiWts`ZOdazmnteLTXjix43NPBF%Q|kz;|P6Mmjb(! zvEVo(OCMvb!=b=;NP-m<9SvUyG3C)up)U1|bz`L&{HtWxls!kQg}>&!b)I{#w=&(jls%Wt8-^_&pL5SSprpaj}g?7f}m$x^fd8K zMy~GShAo@!N|taoHbujRnK*8qJPZU7JU|{yP3=c;Kv+IK*aQ54@lokrVnT?|8#(%Z zyf0b+%EhZz(k6jr7Qe+l67Ve{gHycbZ&;VRjT+ik$-;ltj-l)<9M7t?t{tjsyfE~} z$6~sNQ}TYIRWb8L$xjWe!#7=V`+O$s08tt60Zp+tSNp@?u=o0`$QJVFiH9pP2JU1v z{2eIz8L>xirl>n;F@Try^#)P~YvVe4oUA$Casm;aipWDX&jHM!N=@kT;O17NlReEd zaXE)8l;5#MgIPM0X!}nI$>=bq$+=^Hi`1SeIkG;s`UEhdg30un= zRr}VdLZ!Zq-_GY;o4{$8;Wrl}qBjwV#GEYvSaadZ9`}@&z`%=EWX1y=3i(z4X5oJ2E*5PPwy|Qb*=}`gn4HJX`;bVQm%v)VO-7(zZ!r0&B z-mR{v5Jz*ssR7`nY^FwE>+IR3XJ5)Hnp3{CL+o14mNxA~ex>~H6nlV{n|+wV9BA6u zVkH#7kt>}QRV$~2+HxW1u*&W1JTH8_lfjn_vvOM=d3K@W_iaC{KDV_!aWsEPUkI79 zSo*KQKNZybyRQI%PG?>NDOd1)FIeI5m3i}#xq*L`#KZ#A-oTzuV3_Qby~_=. +*/ + +/** + * @file src/regex/perf-regex.c + * @brief Test how long it takes to create a automaton from a string regex. + * @author Bartlomiej Polot + */ +#include +#include +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" + + +/** + * Print information about the given node and its edges + * to stdout. + * + * @param cls closure, unused. + * @param key hash for current state. + * @param proof proof for current state. + * @param accepting GNUNET_YES if this is an accepting state, GNUNET_NO if not. + * @param num_edges number of edges leaving current state. + * @param edges edges leaving current state. + */ +static void +print_edge (void *cls, + const struct GNUNET_HashCode *key, + const char *proof, + int accepting, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges) +{ + unsigned int i; + + printf ("%s: %s, proof: `%s'\n", + GNUNET_h2s (key), + accepting ? "ACCEPTING" : "", + proof); + for (i = 0; i < num_edges; i++) + printf (" `%s': %s\n", + edges[i].label, + GNUNET_h2s (&edges[i].destination)); +} + + +/** + * The main function of the regex performace test. + * + * Read a set of regex from a file, combine them and create a DFA from the + * resulting combined regex. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + struct REGEX_INTERNAL_Automaton* dfa; + char **regexes; + char *buffer; + char *regex; + int compression; + unsigned int alphabet_size; + long size; + + GNUNET_log_setup ("perf-regex", "DEBUG", NULL); + if (4 != argc) + { + fprintf (stderr, + "Usage: %s REGEX_FILE ALPHABET_SIZE COMPRESSION\n", + argv[0]); + return 1; + } + regexes = REGEX_TEST_read_from_file (argv[1]); + if (NULL == regexes) + { + fprintf (stderr, + "Failed to read regexes from `%s'\n", + argv[1]); + return 2; + } + alphabet_size = atoi (argv[2]); + compression = atoi (argv[3]); + printf ("********* PERF-REGEX *********'\n"); + printf ("Using:\n file '%s'\n Alphabet size %u\n compression %d\n", + argv[1], alphabet_size, compression); + fflush(stdout); + buffer = REGEX_TEST_combine (regexes, alphabet_size); + GNUNET_asprintf (®ex, "GNUNET_REGEX_PROFILER_(%s)(0|1)*", buffer); + size = strlen (regex); + + fprintf (stderr, + "Combined regex (%ld bytes):\n%s\n", + size, + regex); + dfa = REGEX_INTERNAL_construct_dfa (regex, size, compression); + printf ("********* ALL EDGES *********'\n"); + REGEX_INTERNAL_iterate_all_edges (dfa, &print_edge, NULL); + printf ("\n\n********* REACHABLE EDGES *********'\n"); + REGEX_INTERNAL_iterate_reachable_edges (dfa, &print_edge, NULL); + REGEX_INTERNAL_automaton_destroy (dfa); + GNUNET_free (buffer); + REGEX_TEST_free_from_file (regexes); + GNUNET_free (regex); + return 0; +} + +/* end of prof-regex.c */ diff --git a/src/regex/plugin_block_regex.c b/src/regex/plugin_block_regex.c new file mode 100644 index 000000000..76045efcd --- /dev/null +++ b/src/regex/plugin_block_regex.c @@ -0,0 +1,402 @@ +/* + This file is part of GNUnet + Copyright (C) 2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +/** + * @file regex/plugin_block_regex.c + * @brief blocks used for regex storage and search + * @author Bartlomiej Polot + */ +#include "platform.h" +#include "gnunet_block_plugin.h" +#include "gnunet_block_group_lib.h" +#include "block_regex.h" +#include "regex_block_lib.h" +#include "gnunet_signatures.h" + + +/** + * Number of bits we set per entry in the bloomfilter. + * Do not change! + */ +#define BLOOMFILTER_K 16 + + +/** + * How big is the BF we use for REGEX blocks? + */ +#define REGEX_BF_SIZE 8 + + +/** + * Create a new block group. + * + * @param ctx block context in which the block group is created + * @param type type of the block for which we are creating the group + * @param nonce random value used to seed the group creation + * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh + * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh + * @param va variable arguments specific to @a type + * @return block group handle, NULL if block groups are not supported + * by this @a type of block (this is not an error) + */ +static struct GNUNET_BLOCK_Group * +block_plugin_regex_create_group (void *cls, + enum GNUNET_BLOCK_Type type, + uint32_t nonce, + const void *raw_data, + size_t raw_data_size, + va_list va) +{ + unsigned int bf_size; + const char *guard; + + guard = va_arg (va, const char *); + if (0 == strcmp (guard, + "seen-set-size")) + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); + else if (0 == strcmp (guard, + "filter-size")) + bf_size = va_arg (va, unsigned int); + else + { + GNUNET_break (0); + bf_size = REGEX_BF_SIZE; + } + GNUNET_break (NULL == va_arg (va, const char *)); + return GNUNET_BLOCK_GROUP_bf_create (cls, + bf_size, + BLOOMFILTER_K, + type, + nonce, + raw_data, + raw_data_size); +} + + +/** + * Function called to validate a reply or a request of type + * #GNUNET_BLOCK_TYPE_REGEX. + * For request evaluation, pass "NULL" for the reply_block. + * Note that it is assumed that the reply has already been + * matched to the key (and signatures checked) as it would + * be done with the #GNUNET_BLOCK_get_key() function. + * + * @param cls closure + * @param type block type + * @param bg block group to evaluate against + * @param eo control flags + * @param query original query (hash) + * @param xquery extrended query data (can be NULL, depending on type) + * @param xquery_size number of bytes in @a xquery + * @param reply_block response to validate + * @param reply_block_size number of bytes in @a reply_block + * @return characterization of result + */ +static enum GNUNET_BLOCK_EvaluationResult +evaluate_block_regex (void *cls, + enum GNUNET_BLOCK_Type type, + struct GNUNET_BLOCK_Group *bg, + enum GNUNET_BLOCK_EvaluationOptions eo, + const struct GNUNET_HashCode *query, + const void *xquery, + size_t xquery_size, + const void *reply_block, + size_t reply_block_size) +{ + struct GNUNET_HashCode chash; + + if (NULL == reply_block) + { + if (0 != xquery_size) + { + const char *s; + + s = (const char *) xquery; + if ('\0' != s[xquery_size - 1]) /* must be valid 0-terminated string */ + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + } + return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; + } + if (0 != xquery_size) + { + const char *s; + + s = (const char *) xquery; + if ('\0' != s[xquery_size - 1]) /* must be valid 0-terminated string */ + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + } + else if (NULL != query) + { + /* xquery is required for regex GETs, at least an empty string */ + GNUNET_break_op (0); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "type %d, query %p, xquery %p\n", + type, query, xquery); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + switch (REGEX_BLOCK_check (reply_block, + reply_block_size, + query, + xquery)) + { + case GNUNET_SYSERR: + GNUNET_break_op(0); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + case GNUNET_NO: + /* xquery missmatch, can happen */ + return GNUNET_BLOCK_EVALUATION_RESULT_IRRELEVANT; + default: + break; + } + GNUNET_CRYPTO_hash (reply_block, + reply_block_size, + &chash); + if (GNUNET_YES == + GNUNET_BLOCK_GROUP_bf_test_and_set (bg, + &chash)) + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; + return GNUNET_BLOCK_EVALUATION_OK_MORE; +} + + +/** + * Function called to validate a reply or a request of type + * #GNUNET_BLOCK_TYPE_REGEX_ACCEPT. + * For request evaluation, pass "NULL" for the reply_block. + * Note that it is assumed that the reply has already been + * matched to the key (and signatures checked) as it would + * be done with the #GNUNET_BLOCK_get_key() function. + * + * @param cls closure + * @param type block type + * @param bg block group to evaluate against + * @param eo control flags + * @param query original query (hash) + * @param xquery extrended query data (can be NULL, depending on type) + * @param xquery_size number of bytes in @a xquery + * @param reply_block response to validate + * @param reply_block_size number of bytes in @a reply_block + * @return characterization of result + */ +static enum GNUNET_BLOCK_EvaluationResult +evaluate_block_regex_accept (void *cls, + enum GNUNET_BLOCK_Type type, + struct GNUNET_BLOCK_Group *bg, + enum GNUNET_BLOCK_EvaluationOptions eo, + const struct GNUNET_HashCode *query, + const void *xquery, + size_t xquery_size, const void *reply_block, + size_t reply_block_size) +{ + const struct RegexAcceptBlock *rba; + struct GNUNET_HashCode chash; + + if (0 != xquery_size) + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + if (NULL == reply_block) + return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; + if (sizeof (struct RegexAcceptBlock) != reply_block_size) + { + GNUNET_break_op(0); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + } + rba = reply_block; + if (ntohl (rba->purpose.size) != + sizeof (struct GNUNET_CRYPTO_EccSignaturePurpose) + + sizeof (struct GNUNET_TIME_AbsoluteNBO) + + sizeof (struct GNUNET_HashCode)) + { + GNUNET_break_op(0); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + } + if (0 == GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_ntoh (rba->expiration_time)).rel_value_us) + { + /* technically invalid, but can happen without an error, so + we're nice by reporting it as a 'duplicate' */ + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; + } + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_REGEX_ACCEPT, + &rba->purpose, + &rba->signature, + &rba->peer.public_key)) + { + GNUNET_break_op(0); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + } + GNUNET_CRYPTO_hash (reply_block, + reply_block_size, + &chash); + if (GNUNET_YES == + GNUNET_BLOCK_GROUP_bf_test_and_set (bg, + &chash)) + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; + return GNUNET_BLOCK_EVALUATION_OK_MORE; +} + + +/** + * Function called to validate a reply or a request. For + * request evaluation, simply pass "NULL" for the reply_block. + * Note that it is assumed that the reply has already been + * matched to the key (and signatures checked) as it would + * be done with the #GNUNET_BLOCK_get_key() function. + * + * @param cls closure + * @param ctx block context + * @param type block type + * @param bg group to evaluate against + * @param eo control flags + * @param query original query (hash) + * @param xquery extrended query data (can be NULL, depending on type) + * @param xquery_size number of bytes in xquery + * @param reply_block response to validate + * @param reply_block_size number of bytes in reply block + * @return characterization of result + */ +static enum GNUNET_BLOCK_EvaluationResult +block_plugin_regex_evaluate (void *cls, + struct GNUNET_BLOCK_Context *ctx, + enum GNUNET_BLOCK_Type type, + struct GNUNET_BLOCK_Group *bg, + enum GNUNET_BLOCK_EvaluationOptions eo, + const struct GNUNET_HashCode *query, + const void *xquery, + size_t xquery_size, + const void *reply_block, + size_t reply_block_size) +{ + enum GNUNET_BLOCK_EvaluationResult result; + + switch (type) + { + case GNUNET_BLOCK_TYPE_REGEX: + result = evaluate_block_regex (cls, + type, + bg, + eo, + query, + xquery, xquery_size, + reply_block, reply_block_size); + break; + case GNUNET_BLOCK_TYPE_REGEX_ACCEPT: + result = evaluate_block_regex_accept (cls, + type, + bg, + eo, + query, + xquery, xquery_size, + reply_block, reply_block_size); + break; + + default: + result = GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; + } + return result; +} + + +/** + * Function called to obtain the key for a block. + * + * @param cls closure + * @param type block type + * @param block block to get the key for + * @param block_size number of bytes in @a block + * @param key set to the key (query) for the given block + * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported + * (or if extracting a key from a block of this type does not work) + */ +static int +block_plugin_regex_get_key (void *cls, + enum GNUNET_BLOCK_Type type, + const void *block, + size_t block_size, + struct GNUNET_HashCode *key) +{ + switch (type) + { + case GNUNET_BLOCK_TYPE_REGEX: + if (GNUNET_OK != + REGEX_BLOCK_get_key (block, block_size, + key)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + return GNUNET_OK; + case GNUNET_BLOCK_TYPE_REGEX_ACCEPT: + if (sizeof (struct RegexAcceptBlock) != block_size) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + *key = ((struct RegexAcceptBlock *) block)->key; + return GNUNET_OK; + default: + GNUNET_break (0); + return GNUNET_SYSERR; + } +} + + +/** + * Entry point for the plugin. + */ +void * +libgnunet_plugin_block_regex_init (void *cls) +{ + static enum GNUNET_BLOCK_Type types[] = + { + GNUNET_BLOCK_TYPE_REGEX, + GNUNET_BLOCK_TYPE_REGEX_ACCEPT, + GNUNET_BLOCK_TYPE_ANY /* end of list */ + }; + struct GNUNET_BLOCK_PluginFunctions *api; + + api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); + api->evaluate = &block_plugin_regex_evaluate; + api->get_key = &block_plugin_regex_get_key; + api->create_group = &block_plugin_regex_create_group; + api->types = types; + return api; +} + + +/** + * Exit point from the plugin. + */ +void * +libgnunet_plugin_block_regex_done (void *cls) +{ + struct GNUNET_BLOCK_PluginFunctions *api = cls; + + GNUNET_free (api); + return NULL; +} + +/* end of plugin_block_regex.c */ diff --git a/src/regex/regex.conf.in b/src/regex/regex.conf.in new file mode 100644 index 000000000..5e68a43da --- /dev/null +++ b/src/regex/regex.conf.in @@ -0,0 +1,8 @@ +[regex] +START_ON_DEMAND = @START_ON_DEMAND@ +@UNIXONLY@ PORT = 2107 +UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-regex.sock +HOSTNAME = localhost +BINARY = gnunet-service-regex +ACCEPT_FROM = 127.0.0.1; +ACCEPT_FROM6 = ::1; diff --git a/src/regex/regex_api_announce.c b/src/regex/regex_api_announce.c new file mode 100644 index 000000000..e3ad70c6a --- /dev/null +++ b/src/regex/regex_api_announce.c @@ -0,0 +1,184 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2013, 2016 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/regex_api_announce.c + * @brief access regex service to advertise capabilities via regex + * @author Maximilian Szengel + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_protocols.h" +#include "gnunet_util_lib.h" +#include "gnunet_regex_service.h" +#include "regex_ipc.h" + +#define LOG(kind,...) GNUNET_log_from (kind, "regex-api",__VA_ARGS__) + +/** + * Handle to store cached data about a regex announce. + */ +struct GNUNET_REGEX_Announcement +{ + /** + * Connection to the regex service. + */ + struct GNUNET_MQ_Handle *mq; + + /** + * Our configuration. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Message we're sending to the service. + */ + char *regex; + + /** + * Frequency of announcements. + */ + struct GNUNET_TIME_Relative refresh_delay; + + /** + * Number of characters per edge. + */ + uint16_t compression; +}; + + + +/** + * (Re)connect to the REGEX service with the given announcement @a a. + * + * @param a REGEX to announce. + */ +static void +announce_reconnect (struct GNUNET_REGEX_Announcement *a); + + +/** + * We got a disconnect after asking regex to do the announcement. + * Retry. + * + * @param cls the `struct GNUNET_REGEX_Announcement` to retry + * @param error error code + */ +static void +announce_mq_error_handler (void *cls, + enum GNUNET_MQ_Error error) +{ + struct GNUNET_REGEX_Announcement *a = cls; + + GNUNET_MQ_destroy (a->mq); + a->mq = NULL; + announce_reconnect (a); +} + + +/** + * (Re)connect to the REGEX service with the given announcement @a a. + * + * @param a REGEX to announce. + */ +static void +announce_reconnect (struct GNUNET_REGEX_Announcement *a) +{ + struct GNUNET_MQ_Envelope *env; + struct AnnounceMessage *am; + size_t slen; + + a->mq = GNUNET_CLIENT_connect (a->cfg, + "regex", + NULL, + &announce_mq_error_handler, + a); + if (NULL == a->mq) + return; + slen = strlen (a->regex) + 1; + env = GNUNET_MQ_msg_extra (am, + slen, + GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE); + am->compression = htons (a->compression); + am->reserved = htons (0); + am->refresh_delay = GNUNET_TIME_relative_hton (a->refresh_delay); + GNUNET_memcpy (&am[1], + a->regex, + slen); + GNUNET_MQ_send (a->mq, + env); +} + + +/** + * Announce the given peer under the given regular expression. + * + * @param cfg configuration to use + * @param regex Regular expression to announce. + * @param refresh_delay after what delay should the announcement be repeated? + * @param compression How many characters per edge can we squeeze? + * @return Handle to reuse o free cached resources. + * Must be freed by calling #GNUNET_REGEX_announce_cancel(). + */ +struct GNUNET_REGEX_Announcement * +GNUNET_REGEX_announce (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *regex, + struct GNUNET_TIME_Relative refresh_delay, + uint16_t compression) +{ + struct GNUNET_REGEX_Announcement *a; + size_t slen; + + slen = strlen (regex) + 1; + if (slen + sizeof (struct AnnounceMessage) >= GNUNET_MAX_MESSAGE_SIZE) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Regex `%s' is too long!\n"), + regex); + GNUNET_break (0); + return NULL; + } + a = GNUNET_new (struct GNUNET_REGEX_Announcement); + a->cfg = cfg; + a->refresh_delay = refresh_delay; + a->compression = compression; + a->regex = GNUNET_strdup (regex); + announce_reconnect (a); + if (NULL == a->mq) + { + GNUNET_free (a->regex); + GNUNET_free (a); + return NULL; + } + return a; +} + + +/** + * Stop announcing the regex specified by the given handle. + * + * @param a handle returned by a previous #GNUNET_REGEX_announce() call. + */ +void +GNUNET_REGEX_announce_cancel (struct GNUNET_REGEX_Announcement *a) +{ + GNUNET_MQ_destroy (a->mq); + GNUNET_free (a->regex); + GNUNET_free (a); +} + +/* end of regex_api_announce.c */ diff --git a/src/regex/regex_api_search.c b/src/regex/regex_api_search.c new file mode 100644 index 000000000..2e2536a02 --- /dev/null +++ b/src/regex/regex_api_search.c @@ -0,0 +1,248 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2013, 2016 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/regex_api_search.c + * @brief access regex service to discover + * peers using matching strings + * @author Maximilian Szengel + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_protocols.h" +#include "gnunet_util_lib.h" +#include "gnunet_regex_service.h" +#include "regex_ipc.h" + +#define LOG(kind,...) GNUNET_log_from (kind, "regex-api",__VA_ARGS__) + + +/** + * Handle to store data about a regex search. + */ +struct GNUNET_REGEX_Search +{ + /** + * Connection to the regex service. + */ + struct GNUNET_MQ_Handle *mq; + + /** + * Our configuration. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Function to call with results. + */ + GNUNET_REGEX_Found callback; + + /** + * Closure for @e callback. + */ + void *callback_cls; + + /** + * Search string to transmit to the service. + */ + char *string; +}; + + +/** + * (Re)connect to the REGEX service for the given search @a s. + * + * @param s context for the search search for + */ +static void +search_reconnect (struct GNUNET_REGEX_Search *s); + + +/** + * We got a response or disconnect after asking regex + * to do the search. Check it is well-formed. + * + * @param cls the `struct GNUNET_REGEX_Search` to handle reply for + * @param result the message + * @return #GNUNET_SYSERR if @a rm is not well-formed. + */ +static int +check_search_response (void *cls, + const struct ResultMessage *result) +{ + uint16_t size = ntohs (result->header.size) - sizeof (*result); + uint16_t gpl = ntohs (result->get_path_length); + uint16_t ppl = ntohs (result->put_path_length); + + if (size != (gpl + ppl) * sizeof (struct GNUNET_PeerIdentity)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * We got a response or disconnect after asking regex + * to do the search. Handle it. + * + * @param cls the `struct GNUNET_REGEX_Search` to handle reply for + * @param result the message + */ +static void +handle_search_response (void *cls, + const struct ResultMessage *result) +{ + struct GNUNET_REGEX_Search *s = cls; + uint16_t gpl = ntohs (result->get_path_length); + uint16_t ppl = ntohs (result->put_path_length); + const struct GNUNET_PeerIdentity *pid; + + pid = &result->id; + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Got regex result %s\n", + GNUNET_i2s (pid)); + s->callback (s->callback_cls, + pid, + &pid[1], + gpl, + &pid[1 + gpl], + ppl); +} + + +/** + * We got a disconnect after asking regex to do the announcement. + * Retry. + * + * @param cls the `struct GNUNET_REGEX_Search` to retry + * @param error error code + */ +static void +mq_error_handler (void *cls, + enum GNUNET_MQ_Error error) +{ + struct GNUNET_REGEX_Search *s = cls; + + GNUNET_MQ_destroy (s->mq); + s->mq = NULL; + search_reconnect (s); +} + + +/** + * (Re)connect to the REGEX service for the given search @a s. + * + * @param s context for the search search for + */ +static void +search_reconnect (struct GNUNET_REGEX_Search *s) +{ + struct GNUNET_MQ_MessageHandler handlers[] = { + GNUNET_MQ_hd_var_size (search_response, + GNUNET_MESSAGE_TYPE_REGEX_RESULT, + struct ResultMessage, + s), + GNUNET_MQ_handler_end () + }; + size_t slen = strlen (s->string) + 1; + struct GNUNET_MQ_Envelope *env; + struct RegexSearchMessage *rsm; + + GNUNET_assert (NULL == s->mq); + s->mq = GNUNET_CLIENT_connect (s->cfg, + "regex", + handlers, + &mq_error_handler, + s); + if (NULL == s->mq) + return; + env = GNUNET_MQ_msg_extra (rsm, + slen, + GNUNET_MESSAGE_TYPE_REGEX_SEARCH); + GNUNET_memcpy (&rsm[1], + s->string, + slen); + GNUNET_MQ_send (s->mq, + env); +} + + +/** + * Search for a peer offering a regex matching certain string in the DHT. + * The search runs until #GNUNET_REGEX_search_cancel() is called, even if results + * are returned. + * + * @param cfg configuration to use + * @param string String to match against the regexes in the DHT. + * @param callback Callback for found peers. + * @param callback_cls Closure for @c callback. + * @return Handle to stop search and free resources. + * Must be freed by calling #GNUNET_REGEX_search_cancel(). + */ +struct GNUNET_REGEX_Search * +GNUNET_REGEX_search (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *string, + GNUNET_REGEX_Found callback, + void *callback_cls) +{ + struct GNUNET_REGEX_Search *s; + size_t slen = strlen (string) + 1; + + if (slen + sizeof (struct RegexSearchMessage) >= GNUNET_MAX_MESSAGE_SIZE) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Search string `%s' is too long!\n"), + string); + GNUNET_break (0); + return NULL; + } + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Starting regex search for %s\n", + string); + s = GNUNET_new (struct GNUNET_REGEX_Search); + s->cfg = cfg; + s->string = GNUNET_strdup (string); + s->callback = callback; + s->callback_cls = callback_cls; + search_reconnect (s); + if (NULL == s->mq) + { + GNUNET_free (s->string); + GNUNET_free (s); + return NULL; + } + return s; +} + + +/** + * Stop search and free all data used by a #GNUNET_REGEX_search() call. + * + * @param s Handle returned by a previous #GNUNET_REGEX_search() call. + */ +void +GNUNET_REGEX_search_cancel (struct GNUNET_REGEX_Search *s) +{ + GNUNET_MQ_destroy (s->mq); + GNUNET_free (s->string); + GNUNET_free (s); +} + + +/* end of regex_api_search.c */ diff --git a/src/regex/regex_block_lib.c b/src/regex/regex_block_lib.c new file mode 100644 index 000000000..33eaf466f --- /dev/null +++ b/src/regex/regex_block_lib.c @@ -0,0 +1,469 @@ +/* + This file is part of GNUnet. + Copyright (C) 2012,2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @author Bartlomiej Polot + * @file regex/regex_block_lib.c + * @brief functions for manipulating non-accept blocks stored for + * regex in the DHT + */ +#include "platform.h" +#include "regex_block_lib.h" +#include "gnunet_constants.h" + +#define LOG(kind,...) GNUNET_log_from (kind,"regex-bck",__VA_ARGS__) + +GNUNET_NETWORK_STRUCT_BEGIN + +/** + * Information for each edge. + */ +struct EdgeInfo +{ + /** + * Index of the destination of this edge in the + * unique destinations array. + */ + uint16_t destination_index GNUNET_PACKED; + + /** + * Number of bytes the token for this edge takes in the + * token area. + */ + uint16_t token_length GNUNET_PACKED; +}; + + +/** + * @brief Block to announce a regex state. + */ +struct RegexBlock +{ + + /** + * Length of the proof regex string. + */ + uint16_t proof_len GNUNET_PACKED; + + /** + * Is this state an accepting state? + */ + int16_t is_accepting GNUNET_PACKED; + + /** + * Number of edges parting from this state. + */ + uint16_t num_edges GNUNET_PACKED; + + /** + * Nubmer of unique destinations reachable from this state. + */ + uint16_t num_destinations GNUNET_PACKED; + + /* followed by 'struct GNUNET_HashCode[num_destinations]' */ + + /* followed by 'struct EdgeInfo[edge_destination_indices]' */ + + /* followed by 'char proof[n_proof]', NOT 0-terminated */ + + /* followed by 'char tokens[num_edges][edge_info[k].token_length]'; + essentially all of the tokens one after the other in the + order of the edges; tokens are NOT 0-terminated */ + +}; + + +GNUNET_NETWORK_STRUCT_END + + +/** + * Test if this block is marked as being an accept state. + * + * @param block block to test + * @param size number of bytes in block + * @return #GNUNET_YES if the block is accepting, #GNUNET_NO if not + */ +int +GNUNET_BLOCK_is_accepting (const struct RegexBlock *block, + size_t size) +{ + if (size < sizeof (struct RegexBlock)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + return ntohs (block->is_accepting); +} + + +/** + * Check if the given 'proof' matches the given 'key'. + * + * @param proof partial regex of a state + * @param proof_len number of bytes in 'proof' + * @param key hash of a state. + * @return #GNUNET_OK if the proof is valid for the given key. + */ +int +REGEX_BLOCK_check_proof (const char *proof, + size_t proof_len, + const struct GNUNET_HashCode *key) +{ + struct GNUNET_HashCode key_check; + + if ( (NULL == proof) || (NULL == key)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Proof check failed, was NULL.\n"); + return GNUNET_NO; + } + GNUNET_CRYPTO_hash (proof, proof_len, &key_check); + return (0 == + GNUNET_CRYPTO_hash_cmp (key, &key_check)) ? GNUNET_OK : GNUNET_NO; +} + + +/** + * Struct to keep track of the xquery while iterating all the edges in a block. + */ +struct CheckEdgeContext +{ + /** + * Xquery: string we are looking for. + */ + const char *xquery; + + /** + * Has any edge matched the xquery so far? (GNUNET_OK / GNUNET_NO) + */ + int found; + +}; + + +/** + * Iterator over all edges in a block, checking for a presence of a given query. + * + * @param cls Closure, (xquery context). + * @param token Token that follows to next state. + * @param len Lenght of token. + * @param key Hash of next state. + * + * @return #GNUNET_YES, to keep iterating + */ +static int +check_edge (void *cls, + const char *token, + size_t len, + const struct GNUNET_HashCode *key) +{ + struct CheckEdgeContext *ctx = cls; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "edge %.*s [%u]: %s\n", + (int) len, + token, + (unsigned int) len, + GNUNET_h2s (key)); + if (NULL == ctx->xquery) + return GNUNET_YES; + if (strlen (ctx->xquery) < len) + return GNUNET_YES; /* too long */ + if (0 == strncmp (ctx->xquery, token, len)) + ctx->found = GNUNET_OK; + return GNUNET_YES; /* keep checking for malformed data! */ +} + + +/** + * Check if the regex block is well formed, including all edges. + * + * @param block The start of the block. + * @param size The size of the block. + * @param query the query for the block + * @param xquery String describing the edge we are looking for. + * Can be NULL in case this is a put block. + * @return #GNUNET_OK in case it's fine. + * #GNUNET_NO in case the xquery exists and is not found (IRRELEVANT). + * #GNUNET_SYSERR if the block is invalid. + */ +int +REGEX_BLOCK_check (const struct RegexBlock *block, + size_t size, + const struct GNUNET_HashCode *query, + const char *xquery) +{ + struct GNUNET_HashCode key; + struct CheckEdgeContext ctx; + int res; + + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Block check\n"); + if (GNUNET_OK != + REGEX_BLOCK_get_key (block, size, + &key)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + if (NULL != query && + 0 != memcmp (&key, + query, + sizeof (struct GNUNET_HashCode))) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + if ( (GNUNET_YES == ntohs (block->is_accepting)) && + ( (NULL == xquery) || ('\0' == xquery[0]) ) ) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, + " out! Is accepting: %u, xquery %p\n", + ntohs(block->is_accepting), + xquery); + return GNUNET_OK; + } + ctx.xquery = xquery; + ctx.found = GNUNET_NO; + res = REGEX_BLOCK_iterate (block, size, &check_edge, &ctx); + if (GNUNET_SYSERR == res) + return GNUNET_SYSERR; + if (NULL == xquery) + return GNUNET_YES; + LOG (GNUNET_ERROR_TYPE_DEBUG, "Result %d\n", ctx.found); + return ctx.found; +} + + +/** + * Obtain the key that a particular block is to be stored under. + * + * @param block block to get the key from + * @param block_len number of bytes in block + * @param key where to store the key + * @return #GNUNET_OK on success, #GNUNET_SYSERR if the block is malformed + */ +int +REGEX_BLOCK_get_key (const struct RegexBlock *block, + size_t block_len, + struct GNUNET_HashCode *key) +{ + uint16_t len; + const struct GNUNET_HashCode *destinations; + const struct EdgeInfo *edges; + uint16_t num_destinations; + uint16_t num_edges; + size_t total; + + if (block_len < sizeof (struct RegexBlock)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + num_destinations = ntohs (block->num_destinations); + num_edges = ntohs (block->num_edges); + len = ntohs (block->proof_len); + destinations = (const struct GNUNET_HashCode *) &block[1]; + edges = (const struct EdgeInfo *) &destinations[num_destinations]; + total = sizeof (struct RegexBlock) + num_destinations * sizeof (struct GNUNET_HashCode) + num_edges * sizeof (struct EdgeInfo) + len; + if (block_len < total) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + GNUNET_CRYPTO_hash (&edges[num_edges], len, key); + return GNUNET_OK; +} + + +/** + * Iterate over all edges of a block of a regex state. + * + * @param block Block to iterate over. + * @param size Size of @a block. + * @param iterator Function to call on each edge in the block. + * @param iter_cls Closure for the @a iterator. + * @return #GNUNET_SYSERR if an error has been encountered. + * #GNUNET_OK if no error has been encountered. + * Note that if the iterator stops the iteration by returning + * #GNUNET_NO, the block will no longer be checked for further errors. + * The return value will be GNUNET_OK meaning that no errors were + * found until the edge last notified to the iterator, but there might + * be errors in further edges. + */ +int +REGEX_BLOCK_iterate (const struct RegexBlock *block, + size_t size, + REGEX_INTERNAL_EgdeIterator iterator, + void *iter_cls) +{ + uint16_t len; + const struct GNUNET_HashCode *destinations; + const struct EdgeInfo *edges; + const char *aux; + uint16_t num_destinations; + uint16_t num_edges; + size_t total; + unsigned int n; + size_t off; + + LOG (GNUNET_ERROR_TYPE_DEBUG, "Block iterate\n"); + if (size < sizeof (struct RegexBlock)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + num_destinations = ntohs (block->num_destinations); + num_edges = ntohs (block->num_edges); + len = ntohs (block->proof_len); + destinations = (const struct GNUNET_HashCode *) &block[1]; + edges = (const struct EdgeInfo *) &destinations[num_destinations]; + aux = (const char *) &edges[num_edges]; + total = sizeof (struct RegexBlock) + num_destinations * sizeof (struct GNUNET_HashCode) + num_edges * sizeof (struct EdgeInfo) + len; + if (size < total) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + for (n=0;n UINT16_MAX) + { + GNUNET_break (0); + return NULL; + } + unique_destinations = 0; + total = sizeof (struct RegexBlock) + len; + for (i=0;i UINT16_MAX) + { + GNUNET_break (0); + return NULL; + } + total += slen; + for (j=0;j= 1024) + { + GNUNET_break (0); + return NULL; + } + destination_indices[i] = j; + if (j == unique_destinations) + destinations[unique_destinations++] = edges[i].destination; + } + total += num_edges * sizeof (struct EdgeInfo) + unique_destinations * sizeof (struct GNUNET_HashCode); + if (total >= GNUNET_CONSTANTS_MAX_BLOCK_SIZE) + { + GNUNET_break (0); + return NULL; + } + block = GNUNET_malloc (total); + block->proof_len = htons (len); + block->is_accepting = htons (accepting); + block->num_edges = htons (num_edges); + block->num_destinations = htons (unique_destinations); + dests = (struct GNUNET_HashCode *) &block[1]; + GNUNET_memcpy (dests, destinations, sizeof (struct GNUNET_HashCode) * unique_destinations); + edgeinfos = (struct EdgeInfo *) &dests[unique_destinations]; + aux = (char *) &edgeinfos[num_edges]; + off = len; + GNUNET_memcpy (aux, proof, len); + for (i=0;i. +*/ + +/** + * @author Bartlomiej Polot + * @file regex/regex_block_lib.h + * @brief common function to manipulate blocks stored by regex in the DHT + */ + +#ifndef REGEX_BLOCK_LIB_H_ +#define REGEX_BLOCK_LIB_H_ + +#ifdef __cplusplus +extern "C" +{ +#if 0 + /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + +#include "platform.h" +#include "block_regex.h" + + +/** + * Representation of a Regex node (and edges) in the DHT. + */ +struct RegexBlock; + + +/** + * Edge representation. + */ +struct REGEX_BLOCK_Edge +{ + /** + * Label of the edge. FIXME: might want to not consume exactly + * multiples of 8 bits, need length! + */ + const char *label; + + /** + * Destionation of the edge. + */ + struct GNUNET_HashCode destination; +}; + + +/** + * Check if the given 'proof' matches the given 'key'. + * + * @param proof partial regex of a state + * @param proof_len number of bytes in @a proof + * @param key hash of a state. + * @return #GNUNET_OK if the proof is valid for the given key. + */ +int +REGEX_BLOCK_check_proof (const char *proof, + size_t proof_len, + const struct GNUNET_HashCode *key); + + +/** + * Check if the regex block is well formed, including all edges. + * + * @param block The start of the block. + * @param size The size of the @a block. + * @param query the query for the @a block + * @param xquery String describing the edge we are looking for. + * Can be NULL in case this is a put block. + * @return #GNUNET_OK in case it's fine. + * #GNUNET_NO in case the xquery exists and is not found (IRRELEVANT). + * #GNUNET_SYSERR if the block is invalid. + */ +int +REGEX_BLOCK_check (const struct RegexBlock *block, + size_t size, + const struct GNUNET_HashCode *query, + const char *xquery); + + +/* FIXME: might want to use 'struct REGEX_BLOCK_Edge' here instead of 3 arguments! */ + +/** + * Iterator over edges in a block. + * + * @param cls Closure. + * @param token Token that follows to next state. + * @param len Length of token. + * @param key Hash of next state. + * @return #GNUNET_YES if should keep iterating, #GNUNET_NO otherwise. + */ +typedef int +(*REGEX_INTERNAL_EgdeIterator)(void *cls, + const char *token, + size_t len, + const struct GNUNET_HashCode *key); + + +/** + * Iterate over all edges of a block of a regex state. + * + * @param block Block to iterate over. + * @param size Size of block. + * @param iterator Function to call on each edge in the block. + * @param iter_cls Closure for the @a iterator. + * @return #GNUNET_SYSERR if an error has been encountered. + * #GNUNET_OK if no error has been encountered. + * Note that if the iterator stops the iteration by returning + * #GNUNET_NO, the block will no longer be checked for further errors. + * The return value will be #GNUNET_OK meaning that no errors were + * found until the edge last notified to the iterator, but there might + * be errors in further edges. + */ +int +REGEX_BLOCK_iterate (const struct RegexBlock *block, + size_t size, + REGEX_INTERNAL_EgdeIterator iterator, + void *iter_cls); + + +/** + * Obtain the key that a particular block is to be stored under. + * + * @param block block to get the key from + * @param block_len number of bytes in @a block + * @param key where to store the key + * @return #GNUNET_OK on success, #GNUNET_SYSERR if the block is malformed + */ +int +REGEX_BLOCK_get_key (const struct RegexBlock *block, + size_t block_len, + struct GNUNET_HashCode *key); + + +/** + * Test if this block is marked as being an accept state. + * + * @param block block to test + * @param size number of bytes in block + * @return #GNUNET_YES if the block is accepting, #GNUNET_NO if not + */ +int +GNUNET_BLOCK_is_accepting (const struct RegexBlock *block, + size_t block_len); + + +/** + * Construct a regex block to be stored in the DHT. + * + * @param proof proof string for the block + * @param num_edges number of edges in the block + * @param edges the edges of the block + * @param accepting is this an accepting state + * @param rsize set to the size of the returned block (OUT-only) + * @return the regex block, NULL on error + */ +struct RegexBlock * +REGEX_BLOCK_create (const char *proof, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges, + int accepting, + size_t *rsize); + + +#if 0 /* keep Emacsens' auto-indent happy */ +{ +#endif +#ifdef __cplusplus +} +#endif + +/* ifndef REGEX_BLOCK_LIB_H */ +#endif +/* end of regex_block_lib.h */ diff --git a/src/regex/regex_internal.c b/src/regex/regex_internal.c new file mode 100644 index 000000000..944ca9bb9 --- /dev/null +++ b/src/regex/regex_internal.c @@ -0,0 +1,3706 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file src/regex/regex_internal.c + * @brief library to create Deterministic Finite Automatons (DFAs) from regular + * expressions (regexes). + * @author Maximilian Szengel + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_regex_service.h" +#include "regex_internal_lib.h" +#include "regex_internal.h" + + +/** + * Set this to #GNUNET_YES to enable state naming. Used to debug NFA->DFA + * creation. Disabled by default for better performance. + */ +#define REGEX_DEBUG_DFA GNUNET_NO + +/** + * Set of states using MDLL API. + */ +struct REGEX_INTERNAL_StateSet_MDLL +{ + /** + * MDLL of states. + */ + struct REGEX_INTERNAL_State *head; + + /** + * MDLL of states. + */ + struct REGEX_INTERNAL_State *tail; + + /** + * Length of the MDLL. + */ + unsigned int len; +}; + + +/** + * Append state to the given StateSet. + * + * @param set set to be modified + * @param state state to be appended + */ +static void +state_set_append (struct REGEX_INTERNAL_StateSet *set, + struct REGEX_INTERNAL_State *state) +{ + if (set->off == set->size) + GNUNET_array_grow (set->states, set->size, set->size * 2 + 4); + set->states[set->off++] = state; +} + + +/** + * Compare two strings for equality. If either is NULL they are not equal. + * + * @param str1 first string for comparison. + * @param str2 second string for comparison. + * + * @return 0 if the strings are the same or both NULL, 1 or -1 if not. + */ +static int +nullstrcmp (const char *str1, const char *str2) +{ + if ((NULL == str1) != (NULL == str2)) + return -1; + if ((NULL == str1) && (NULL == str2)) + return 0; + + return strcmp (str1, str2); +} + + +/** + * Adds a transition from one state to another on @a label. Does not add + * duplicate states. + * + * @param ctx context + * @param from_state starting state for the transition + * @param label transition label + * @param to_state state to where the transition should point to + */ +static void +state_add_transition (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_State *from_state, + const char *label, + struct REGEX_INTERNAL_State *to_state) +{ + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_Transition *oth; + + if (NULL == from_state) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not create Transition.\n"); + return; + } + + /* Do not add duplicate state transitions */ + for (t = from_state->transitions_head; NULL != t; t = t->next) + { + if (t->to_state == to_state && 0 == nullstrcmp (t->label, label) && + t->from_state == from_state) + return; + } + + /* sort transitions by label */ + for (oth = from_state->transitions_head; NULL != oth; oth = oth->next) + { + if (0 < nullstrcmp (oth->label, label)) + break; + } + + t = GNUNET_new (struct REGEX_INTERNAL_Transition); + if (NULL != ctx) + t->id = ctx->transition_id++; + if (NULL != label) + t->label = GNUNET_strdup (label); + else + t->label = NULL; + t->to_state = to_state; + t->from_state = from_state; + + /* Add outgoing transition to 'from_state' */ + from_state->transition_count++; + GNUNET_CONTAINER_DLL_insert_before (from_state->transitions_head, + from_state->transitions_tail, oth, t); +} + + +/** + * Remove a 'transition' from 'state'. + * + * @param state state from which the to-be-removed transition originates. + * @param transition transition that should be removed from state 'state'. + */ +static void +state_remove_transition (struct REGEX_INTERNAL_State *state, + struct REGEX_INTERNAL_Transition *transition) +{ + if (NULL == state || NULL == transition) + return; + + if (transition->from_state != state) + return; + + GNUNET_free_non_null (transition->label); + + state->transition_count--; + GNUNET_CONTAINER_DLL_remove (state->transitions_head, state->transitions_tail, + transition); + + GNUNET_free (transition); +} + + +/** + * Compare two states. Used for sorting. + * + * @param a first state + * @param b second state + * + * @return an integer less than, equal to, or greater than zero + * if the first argument is considered to be respectively + * less than, equal to, or greater than the second. + */ +static int +state_compare (const void *a, const void *b) +{ + struct REGEX_INTERNAL_State **s1 = (struct REGEX_INTERNAL_State **) a; + struct REGEX_INTERNAL_State **s2 = (struct REGEX_INTERNAL_State **) b; + + return (*s1)->id - (*s2)->id; +} + + +/** + * Get all edges leaving state @a s. + * + * @param s state. + * @param edges all edges leaving @a s, expected to be allocated and have enough + * space for `s->transitions_count` elements. + * + * @return number of edges. + */ +static unsigned int +state_get_edges (struct REGEX_INTERNAL_State *s, + struct REGEX_BLOCK_Edge *edges) +{ + struct REGEX_INTERNAL_Transition *t; + unsigned int count; + + if (NULL == s) + return 0; + + count = 0; + + for (t = s->transitions_head; NULL != t; t = t->next) + { + if (NULL != t->to_state) + { + edges[count].label = t->label; + edges[count].destination = t->to_state->hash; + count++; + } + } + return count; +} + + +/** + * Compare to state sets by comparing the id's of the states that are contained + * in each set. Both sets are expected to be sorted by id! + * + * @param sset1 first state set + * @param sset2 second state set + * @return 0 if the sets are equal, otherwise non-zero + */ +static int +state_set_compare (struct REGEX_INTERNAL_StateSet *sset1, + struct REGEX_INTERNAL_StateSet *sset2) +{ + int result; + unsigned int i; + + if (NULL == sset1 || NULL == sset2) + return 1; + + result = sset1->off - sset2->off; + if (result < 0) + return -1; + if (result > 0) + return 1; + for (i = 0; i < sset1->off; i++) + if (0 != (result = state_compare (&sset1->states[i], &sset2->states[i]))) + break; + return result; +} + + +/** + * Clears the given StateSet 'set' + * + * @param set set to be cleared + */ +static void +state_set_clear (struct REGEX_INTERNAL_StateSet *set) +{ + GNUNET_array_grow (set->states, set->size, 0); + set->off = 0; +} + + +/** + * Clears an automaton fragment. Does not destroy the states inside the + * automaton. + * + * @param a automaton to be cleared + */ +static void +automaton_fragment_clear (struct REGEX_INTERNAL_Automaton *a) +{ + if (NULL == a) + return; + + a->start = NULL; + a->end = NULL; + a->states_head = NULL; + a->states_tail = NULL; + a->state_count = 0; + GNUNET_free (a); +} + + +/** + * Frees the memory used by State @a s + * + * @param s state that should be destroyed + */ +static void +automaton_destroy_state (struct REGEX_INTERNAL_State *s) +{ + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_Transition *next_t; + + if (NULL == s) + return; + + GNUNET_free_non_null (s->name); + GNUNET_free_non_null (s->proof); + state_set_clear (&s->nfa_set); + for (t = s->transitions_head; NULL != t; t = next_t) + { + next_t = t->next; + state_remove_transition (s, t); + } + + GNUNET_free (s); +} + + +/** + * Remove a state from the given automaton 'a'. Always use this function when + * altering the states of an automaton. Will also remove all transitions leading + * to this state, before destroying it. + * + * @param a automaton + * @param s state to remove + */ +static void +automaton_remove_state (struct REGEX_INTERNAL_Automaton *a, + struct REGEX_INTERNAL_State *s) +{ + struct REGEX_INTERNAL_State *s_check; + struct REGEX_INTERNAL_Transition *t_check; + struct REGEX_INTERNAL_Transition *t_check_next; + + if (NULL == a || NULL == s) + return; + + /* remove all transitions leading to this state */ + for (s_check = a->states_head; NULL != s_check; s_check = s_check->next) + { + for (t_check = s_check->transitions_head; NULL != t_check; + t_check = t_check_next) + { + t_check_next = t_check->next; + if (t_check->to_state == s) + state_remove_transition (s_check, t_check); + } + } + + /* remove state */ + GNUNET_CONTAINER_DLL_remove (a->states_head, a->states_tail, s); + a->state_count--; + + automaton_destroy_state (s); +} + + +/** + * Merge two states into one. Will merge 's1' and 's2' into 's1' and destroy + * 's2'. 's1' will contain all (non-duplicate) outgoing transitions of 's2'. + * + * @param ctx context + * @param a automaton + * @param s1 first state + * @param s2 second state, will be destroyed + */ +static void +automaton_merge_states (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_Automaton *a, + struct REGEX_INTERNAL_State *s1, + struct REGEX_INTERNAL_State *s2) +{ + struct REGEX_INTERNAL_State *s_check; + struct REGEX_INTERNAL_Transition *t_check; + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_Transition *t_next; + int is_dup; + + if (s1 == s2) + return; + + /* 1. Make all transitions pointing to s2 point to s1, unless this transition + * does not already exists, if it already exists remove transition. */ + for (s_check = a->states_head; NULL != s_check; s_check = s_check->next) + { + for (t_check = s_check->transitions_head; NULL != t_check; t_check = t_next) + { + t_next = t_check->next; + + if (s2 == t_check->to_state) + { + is_dup = GNUNET_NO; + for (t = t_check->from_state->transitions_head; NULL != t; t = t->next) + { + if (t->to_state == s1 && 0 == strcmp (t_check->label, t->label)) + is_dup = GNUNET_YES; + } + if (GNUNET_NO == is_dup) + t_check->to_state = s1; + else + state_remove_transition (t_check->from_state, t_check); + } + } + } + + /* 2. Add all transitions from s2 to sX to s1 */ + for (t_check = s2->transitions_head; NULL != t_check; t_check = t_check->next) + { + if (t_check->to_state != s1) + state_add_transition (ctx, s1, t_check->label, t_check->to_state); + } + + /* 3. Rename s1 to {s1,s2} */ +#if REGEX_DEBUG_DFA + char *new_name; + + new_name = s1->name; + GNUNET_asprintf (&s1->name, "{%s,%s}", new_name, s2->name); + GNUNET_free (new_name); +#endif + + /* remove state */ + GNUNET_CONTAINER_DLL_remove (a->states_head, a->states_tail, s2); + a->state_count--; + automaton_destroy_state (s2); +} + + +/** + * Add a state to the automaton 'a', always use this function to alter the + * states DLL of the automaton. + * + * @param a automaton to add the state to + * @param s state that should be added + */ +static void +automaton_add_state (struct REGEX_INTERNAL_Automaton *a, + struct REGEX_INTERNAL_State *s) +{ + GNUNET_CONTAINER_DLL_insert (a->states_head, a->states_tail, s); + a->state_count++; +} + + +/** + * Depth-first traversal (DFS) of all states that are reachable from state + * 's'. Performs 'action' on each visited state. + * + * @param s start state. + * @param marks an array of size a->state_count to remember which state was + * already visited. + * @param count current count of the state. + * @param check function that is checked before advancing on each transition + * in the DFS. + * @param check_cls closure for check. + * @param action action to be performed on each state. + * @param action_cls closure for action. + */ +static void +automaton_state_traverse (struct REGEX_INTERNAL_State *s, int *marks, + unsigned int *count, + REGEX_INTERNAL_traverse_check check, void *check_cls, + REGEX_INTERNAL_traverse_action action, void *action_cls) +{ + struct REGEX_INTERNAL_Transition *t; + + if (GNUNET_YES == marks[s->traversal_id]) + return; + + marks[s->traversal_id] = GNUNET_YES; + + if (NULL != action) + action (action_cls, *count, s); + + (*count)++; + + for (t = s->transitions_head; NULL != t; t = t->next) + { + if (NULL == check || + (NULL != check && GNUNET_YES == check (check_cls, s, t))) + { + automaton_state_traverse (t->to_state, marks, count, check, check_cls, + action, action_cls); + } + } +} + + +/** + * Traverses the given automaton using depth-first-search (DFS) from it's start + * state, visiting all reachable states and calling 'action' on each one of + * them. + * + * @param a automaton to be traversed. + * @param start start state, pass a->start or NULL to traverse the whole automaton. + * @param check function that is checked before advancing on each transition + * in the DFS. + * @param check_cls closure for @a check. + * @param action action to be performed on each state. + * @param action_cls closure for @a action + */ +void +REGEX_INTERNAL_automaton_traverse (const struct REGEX_INTERNAL_Automaton *a, + struct REGEX_INTERNAL_State *start, + REGEX_INTERNAL_traverse_check check, + void *check_cls, + REGEX_INTERNAL_traverse_action action, + void *action_cls) +{ + unsigned int count; + struct REGEX_INTERNAL_State *s; + + if (NULL == a || 0 == a->state_count) + return; + + int marks[a->state_count]; + + for (count = 0, s = a->states_head; NULL != s && count < a->state_count; + s = s->next, count++) + { + s->traversal_id = count; + marks[s->traversal_id] = GNUNET_NO; + } + + count = 0; + + if (NULL == start) + s = a->start; + else + s = start; + + automaton_state_traverse (s, marks, &count, + check, check_cls, + action, action_cls); +} + + +/** + * String container for faster string operations. + */ +struct StringBuffer +{ + /** + * Buffer holding the string (may start in the middle!); + * NOT 0-terminated! + */ + char *sbuf; + + /** + * Allocated buffer. + */ + char *abuf; + + /** + * Length of the string in the buffer. + */ + size_t slen; + + /** + * Number of bytes allocated for @e sbuf + */ + unsigned int blen; + + /** + * Buffer currently represents "NULL" (not the empty string!) + */ + int16_t null_flag; + + /** + * If this entry is part of the last/current generation array, + * this flag is #GNUNET_YES if the last and current generation are + * identical (and thus copying is unnecessary if the value didn't + * change). This is used in an optimization that improves + * performance by about 1% --- if we use int16_t here. With just + * "int" for both flags, performance drops (on my system) significantly, + * most likely due to increased cache misses. + */ + int16_t synced; + +}; + + +/** + * Compare two strings for equality. If either is NULL they are not equal. + * + * @param s1 first string for comparison. + * @param s2 second string for comparison. + * + * @return 0 if the strings are the same or both NULL, 1 or -1 if not. + */ +static int +sb_nullstrcmp (const struct StringBuffer *s1, + const struct StringBuffer *s2) +{ + if ( (GNUNET_YES == s1->null_flag) && + (GNUNET_YES == s2->null_flag) ) + return 0; + if ( (GNUNET_YES == s1->null_flag) || + (GNUNET_YES == s2->null_flag) ) + return -1; + if (s1->slen != s2->slen) + return -1; + if (0 == s1->slen) + return 0; + return memcmp (s1->sbuf, s2->sbuf, s1->slen); +} + + +/** + * Compare two strings for equality. + * + * @param s1 first string for comparison. + * @param s2 second string for comparison. + * + * @return 0 if the strings are the same, 1 or -1 if not. + */ +static int +sb_strcmp (const struct StringBuffer *s1, + const struct StringBuffer *s2) +{ + if (s1->slen != s2->slen) + return -1; + if (0 == s1->slen) + return 0; + return memcmp (s1->sbuf, s2->sbuf, s1->slen); +} + + +/** + * Reallocate the buffer of 'ret' to fit 'nlen' characters; + * move the existing string to the beginning of the new buffer. + * + * @param ret current buffer, to be updated + * @param nlen target length for the buffer, must be at least ret->slen + */ +static void +sb_realloc (struct StringBuffer *ret, + size_t nlen) +{ + char *old; + + GNUNET_assert (nlen >= ret->slen); + old = ret->abuf; + ret->abuf = GNUNET_malloc (nlen); + ret->blen = nlen; + GNUNET_memcpy (ret->abuf, + ret->sbuf, + ret->slen); + ret->sbuf = ret->abuf; + GNUNET_free_non_null (old); +} + + +/** + * Append a string. + * + * @param ret where to write the result + * @param sarg string to append + */ +static void +sb_append (struct StringBuffer *ret, + const struct StringBuffer *sarg) +{ + if (GNUNET_YES == ret->null_flag) + ret->slen = 0; + ret->null_flag = GNUNET_NO; + if (ret->blen < sarg->slen + ret->slen) + sb_realloc (ret, ret->blen + sarg->slen + 128); + GNUNET_memcpy (&ret->sbuf[ret->slen], + sarg->sbuf, + sarg->slen); + ret->slen += sarg->slen; +} + + +/** + * Append a C string. + * + * @param ret where to write the result + * @param cstr string to append + */ +static void +sb_append_cstr (struct StringBuffer *ret, + const char *cstr) +{ + size_t cstr_len = strlen (cstr); + + if (GNUNET_YES == ret->null_flag) + ret->slen = 0; + ret->null_flag = GNUNET_NO; + if (ret->blen < cstr_len + ret->slen) + sb_realloc (ret, ret->blen + cstr_len + 128); + GNUNET_memcpy (&ret->sbuf[ret->slen], + cstr, + cstr_len); + ret->slen += cstr_len; +} + + +/** + * Wrap a string buffer, that is, set ret to the format string + * which contains an "%s" which is to be replaced with the original + * content of 'ret'. Note that optimizing this function is not + * really worth it, it is rarely called. + * + * @param ret where to write the result and take the input for %.*s from + * @param format format string, fprintf-style, with exactly one "%.*s" + * @param extra_chars how long will the result be, in addition to 'sarg' length + */ +static void +sb_wrap (struct StringBuffer *ret, + const char *format, + size_t extra_chars) +{ + char *temp; + + if (GNUNET_YES == ret->null_flag) + ret->slen = 0; + ret->null_flag = GNUNET_NO; + temp = GNUNET_malloc (ret->slen + extra_chars + 1); + GNUNET_snprintf (temp, + ret->slen + extra_chars + 1, + format, + (int) ret->slen, + ret->sbuf); + GNUNET_free_non_null (ret->abuf); + ret->abuf = temp; + ret->sbuf = temp; + ret->blen = ret->slen + extra_chars + 1; + ret->slen = ret->slen + extra_chars; +} + + +/** + * Format a string buffer. Note that optimizing this function is not + * really worth it, it is rarely called. + * + * @param ret where to write the result + * @param format format string, fprintf-style, with exactly one "%.*s" + * @param extra_chars how long will the result be, in addition to 'sarg' length + * @param sarg string to print into the format + */ +static void +sb_printf1 (struct StringBuffer *ret, + const char *format, + size_t extra_chars, + const struct StringBuffer *sarg) +{ + if (ret->blen < sarg->slen + extra_chars + 1) + sb_realloc (ret, + sarg->slen + extra_chars + 1); + ret->null_flag = GNUNET_NO; + ret->sbuf = ret->abuf; + ret->slen = sarg->slen + extra_chars; + GNUNET_snprintf (ret->sbuf, + ret->blen, + format, + (int) sarg->slen, + sarg->sbuf); +} + + +/** + * Format a string buffer. + * + * @param ret where to write the result + * @param format format string, fprintf-style, with exactly two "%.*s" + * @param extra_chars how long will the result be, in addition to 'sarg1/2' length + * @param sarg1 first string to print into the format + * @param sarg2 second string to print into the format + */ +static void +sb_printf2 (struct StringBuffer *ret, + const char *format, + size_t extra_chars, + const struct StringBuffer *sarg1, + const struct StringBuffer *sarg2) +{ + if (ret->blen < sarg1->slen + sarg2->slen + extra_chars + 1) + sb_realloc (ret, + sarg1->slen + sarg2->slen + extra_chars + 1); + ret->null_flag = GNUNET_NO; + ret->slen = sarg1->slen + sarg2->slen + extra_chars; + ret->sbuf = ret->abuf; + GNUNET_snprintf (ret->sbuf, + ret->blen, + format, + (int) sarg1->slen, + sarg1->sbuf, + (int) sarg2->slen, + sarg2->sbuf); +} + + +/** + * Format a string buffer. Note that optimizing this function is not + * really worth it, it is rarely called. + * + * @param ret where to write the result + * @param format format string, fprintf-style, with exactly three "%.*s" + * @param extra_chars how long will the result be, in addition to 'sarg1/2/3' length + * @param sarg1 first string to print into the format + * @param sarg2 second string to print into the format + * @param sarg3 third string to print into the format + */ +static void +sb_printf3 (struct StringBuffer *ret, + const char *format, + size_t extra_chars, + const struct StringBuffer *sarg1, + const struct StringBuffer *sarg2, + const struct StringBuffer *sarg3) +{ + if (ret->blen < sarg1->slen + sarg2->slen + sarg3->slen + extra_chars + 1) + sb_realloc (ret, + sarg1->slen + sarg2->slen + sarg3->slen + extra_chars + 1); + ret->null_flag = GNUNET_NO; + ret->slen = sarg1->slen + sarg2->slen + sarg3->slen + extra_chars; + ret->sbuf = ret->abuf; + GNUNET_snprintf (ret->sbuf, + ret->blen, + format, + (int) sarg1->slen, + sarg1->sbuf, + (int) sarg2->slen, + sarg2->sbuf, + (int) sarg3->slen, + sarg3->sbuf); +} + + +/** + * Free resources of the given string buffer. + * + * @param sb buffer to free (actual pointer is not freed, as they + * should not be individually allocated) + */ +static void +sb_free (struct StringBuffer *sb) +{ + GNUNET_array_grow (sb->abuf, + sb->blen, + 0); + sb->slen = 0; + sb->sbuf = NULL; + sb->null_flag= GNUNET_YES; +} + + +/** + * Copy the given string buffer from 'in' to 'out'. + * + * @param in input string + * @param out output string + */ +static void +sb_strdup (struct StringBuffer *out, + const struct StringBuffer *in) + +{ + out->null_flag = in->null_flag; + if (GNUNET_YES == out->null_flag) + return; + if (out->blen < in->slen) + { + GNUNET_array_grow (out->abuf, + out->blen, + in->slen); + } + out->sbuf = out->abuf; + out->slen = in->slen; + GNUNET_memcpy (out->sbuf, in->sbuf, out->slen); +} + + +/** + * Copy the given string buffer from 'in' to 'out'. + * + * @param cstr input string + * @param out output string + */ +static void +sb_strdup_cstr (struct StringBuffer *out, + const char *cstr) +{ + if (NULL == cstr) + { + out->null_flag = GNUNET_YES; + return; + } + out->null_flag = GNUNET_NO; + out->slen = strlen (cstr); + if (out->blen < out->slen) + { + GNUNET_array_grow (out->abuf, + out->blen, + out->slen); + } + out->sbuf = out->abuf; + GNUNET_memcpy (out->sbuf, cstr, out->slen); +} + + +/** + * Check if the given string @a str needs parentheses around it when + * using it to generate a regex. + * + * @param str string + * + * @return #GNUNET_YES if parentheses are needed, #GNUNET_NO otherwise + */ +static int +needs_parentheses (const struct StringBuffer *str) +{ + size_t slen; + const char *op; + const char *cl; + const char *pos; + const char *end; + unsigned int cnt; + + if ((GNUNET_YES == str->null_flag) || ((slen = str->slen) < 2)) + return GNUNET_NO; + pos = str->sbuf; + if ('(' != pos[0]) + return GNUNET_YES; + end = str->sbuf + slen; + cnt = 1; + pos++; + while (cnt > 0) + { + cl = memchr (pos, ')', end - pos); + if (NULL == cl) + { + GNUNET_break (0); + return GNUNET_YES; + } + /* while '(' before ')', count opening parens */ + while ( (NULL != (op = memchr (pos, '(', end - pos))) && + (op < cl) ) + { + cnt++; + pos = op + 1; + } + /* got ')' first */ + cnt--; + pos = cl + 1; + } + return (*pos == '\0') ? GNUNET_NO : GNUNET_YES; +} + + +/** + * Remove parentheses surrounding string @a str. + * Example: "(a)" becomes "a", "(a|b)|(a|c)" stays the same. + * You need to #GNUNET_free() the returned string. + * + * @param str string, modified to contain a + * @return string without surrounding parentheses, string 'str' if no preceding + * epsilon could be found, NULL if 'str' was NULL + */ +static void +remove_parentheses (struct StringBuffer *str) +{ + size_t slen; + const char *pos; + const char *end; + const char *sbuf; + const char *op; + const char *cp; + unsigned int cnt; + + if (0) + return; + sbuf = str->sbuf; + if ( (GNUNET_YES == str->null_flag) || + (1 >= (slen = str->slen)) || + ('(' != str->sbuf[0]) || + (')' != str->sbuf[slen - 1]) ) + return; + cnt = 0; + pos = &sbuf[1]; + end = &sbuf[slen - 1]; + op = memchr (pos, '(', end - pos); + cp = memchr (pos, ')', end - pos); + while (NULL != cp) + { + while ( (NULL != op) && + (op < cp) ) + { + cnt++; + pos = op + 1; + op = memchr (pos, '(', end - pos); + } + while ( (NULL != cp) && + ( (NULL == op) || + (cp < op) ) ) + { + if (0 == cnt) + return; /* can't strip parens */ + cnt--; + pos = cp + 1; + cp = memchr (pos, ')', end - pos); + } + } + if (0 != cnt) + { + GNUNET_break (0); + return; + } + str->sbuf++; + str->slen -= 2; +} + + +/** + * Check if the string 'str' starts with an epsilon (empty string). + * Example: "(|a)" is starting with an epsilon. + * + * @param str string to test + * + * @return 0 if str has no epsilon, 1 if str starts with '(|' and ends with ')' + */ +static int +has_epsilon (const struct StringBuffer *str) +{ + return + (GNUNET_YES != str->null_flag) && + (0 < str->slen) && + ('(' == str->sbuf[0]) && + ('|' == str->sbuf[1]) && + (')' == str->sbuf[str->slen - 1]); +} + + +/** + * Remove an epsilon from the string str. Where epsilon is an empty string + * Example: str = "(|a|b|c)", result: "a|b|c" + * The returned string needs to be freed. + * + * @param str original string + * @param ret where to return string without preceding epsilon, string 'str' if no preceding + * epsilon could be found, NULL if 'str' was NULL + */ +static void +remove_epsilon (const struct StringBuffer *str, + struct StringBuffer *ret) +{ + if (GNUNET_YES == str->null_flag) + { + ret->null_flag = GNUNET_YES; + return; + } + if ( (str->slen > 1) && + ('(' == str->sbuf[0]) && + ('|' == str->sbuf[1]) && + (')' == str->sbuf[str->slen - 1]) ) + { + /* remove epsilon */ + if (ret->blen < str->slen - 3) + { + GNUNET_array_grow (ret->abuf, + ret->blen, + str->slen - 3); + } + ret->sbuf = ret->abuf; + ret->slen = str->slen - 3; + GNUNET_memcpy (ret->sbuf, &str->sbuf[2], ret->slen); + return; + } + sb_strdup (ret, str); +} + + +/** + * Compare n bytes of 'str1' and 'str2' + * + * @param str1 first string to compare + * @param str2 second string for comparison + * @param n number of bytes to compare + * + * @return -1 if any of the strings is NULL, 0 if equal, non 0 otherwise + */ +static int +sb_strncmp (const struct StringBuffer *str1, + const struct StringBuffer *str2, size_t n) +{ + size_t max; + + if ( (str1->slen != str2->slen) && + ( (str1->slen < n) || + (str2->slen < n) ) ) + return -1; + max = GNUNET_MAX (str1->slen, str2->slen); + if (max > n) + max = n; + return memcmp (str1->sbuf, str2->sbuf, max); +} + + +/** + * Compare n bytes of 'str1' and 'str2' + * + * @param str1 first string to compare + * @param str2 second C string for comparison + * @param n number of bytes to compare (and length of str2) + * + * @return -1 if any of the strings is NULL, 0 if equal, non 0 otherwise + */ +static int +sb_strncmp_cstr (const struct StringBuffer *str1, + const char *str2, size_t n) +{ + if (str1->slen < n) + return -1; + return memcmp (str1->sbuf, str2, n); +} + + +/** + * Initialize string buffer for storing strings of up to n + * characters. + * + * @param sb buffer to initialize + * @param n desired target length + */ +static void +sb_init (struct StringBuffer *sb, + size_t n) +{ + sb->null_flag = GNUNET_NO; + sb->abuf = sb->sbuf = (0 == n) ? NULL : GNUNET_malloc (n); + sb->blen = n; + sb->slen = 0; +} + + +/** + * Compare 'str1', starting from position 'k', with whole 'str2' + * + * @param str1 first string to compare, starting from position 'k' + * @param str2 second string for comparison + * @param k starting position in 'str1' + * + * @return -1 if any of the strings is NULL, 0 if equal, non 0 otherwise + */ +static int +sb_strkcmp (const struct StringBuffer *str1, + const struct StringBuffer *str2, size_t k) +{ + if ( (GNUNET_YES == str1->null_flag) || + (GNUNET_YES == str2->null_flag) || + (k > str1->slen) || + (str1->slen - k != str2->slen) ) + return -1; + return memcmp (&str1->sbuf[k], str2->sbuf, str2->slen); +} + + +/** + * Helper function used as 'action' in 'REGEX_INTERNAL_automaton_traverse' + * function to create the depth-first numbering of the states. + * + * @param cls states array. + * @param count current state counter. + * @param s current state. + */ +static void +number_states (void *cls, const unsigned int count, + struct REGEX_INTERNAL_State *s) +{ + struct REGEX_INTERNAL_State **states = cls; + + s->dfs_id = count; + if (NULL != states) + states[count] = s; +} + + + +#define PRIS(a) \ + ((GNUNET_YES == a.null_flag) ? 6 : (int) a.slen), \ + ((GNUNET_YES == a.null_flag) ? "(null)" : a.sbuf) + + +/** + * Construct the regular expression given the inductive step, + * $R^{(k)}_{ij} = R^{(k-1)}_{ij} | R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* + * R^{(k-1)}_{kj}, and simplify the resulting expression saved in R_cur_ij. + * + * @param R_last_ij value of $R^{(k-1)_{ij}. + * @param R_last_ik value of $R^{(k-1)_{ik}. + * @param R_last_kk value of $R^{(k-1)_{kk}. + * @param R_last_kj value of $R^{(k-1)_{kj}. + * @param R_cur_ij result for this inductive step is saved in R_cur_ij, R_cur_ij + * is expected to be NULL when called! + * @param R_cur_l optimization -- kept between iterations to avoid realloc + * @param R_cur_r optimization -- kept between iterations to avoid realloc + */ +static void +automaton_create_proofs_simplify (const struct StringBuffer *R_last_ij, + const struct StringBuffer *R_last_ik, + const struct StringBuffer *R_last_kk, + const struct StringBuffer *R_last_kj, + struct StringBuffer *R_cur_ij, + struct StringBuffer *R_cur_l, + struct StringBuffer *R_cur_r) +{ + struct StringBuffer R_temp_ij; + struct StringBuffer R_temp_ik; + struct StringBuffer R_temp_kj; + struct StringBuffer R_temp_kk; + int eps_check; + int ij_ik_cmp; + int ij_kj_cmp; + int ik_kk_cmp; + int kk_kj_cmp; + int clean_ik_kk_cmp; + int clean_kk_kj_cmp; + size_t length; + size_t length_l; + size_t length_r; + + /* + * $R^{(k)}_{ij} = R^{(k-1)}_{ij} | R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* R^{(k-1)}_{kj} + * R_last == R^{(k-1)}, R_cur == R^{(k)} + * R_cur_ij = R_cur_l | R_cur_r + * R_cur_l == R^{(k-1)}_{ij} + * R_cur_r == R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* R^{(k-1)}_{kj} + */ + + if ( (GNUNET_YES == R_last_ij->null_flag) && + ( (GNUNET_YES == R_last_ik->null_flag) || + (GNUNET_YES == R_last_kj->null_flag))) + { + /* R^{(k)}_{ij} = N | N */ + R_cur_ij->null_flag = GNUNET_YES; + R_cur_ij->synced = GNUNET_NO; + return; + } + + if ( (GNUNET_YES == R_last_ik->null_flag) || + (GNUNET_YES == R_last_kj->null_flag) ) + { + /* R^{(k)}_{ij} = R^{(k-1)}_{ij} | N */ + if (GNUNET_YES == R_last_ij->synced) + { + R_cur_ij->synced = GNUNET_YES; + R_cur_ij->null_flag = GNUNET_NO; + return; + } + R_cur_ij->synced = GNUNET_YES; + sb_strdup (R_cur_ij, R_last_ij); + return; + } + R_cur_ij->synced = GNUNET_NO; + + /* $R^{(k)}_{ij} = N | R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* R^{(k-1)}_{kj} OR + * $R^{(k)}_{ij} = R^{(k-1)}_{ij} | R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* R^{(k-1)}_{kj} */ + + R_cur_r->null_flag = GNUNET_YES; + R_cur_r->slen = 0; + R_cur_l->null_flag = GNUNET_YES; + R_cur_l->slen = 0; + + /* cache results from strcmp, we might need these many times */ + ij_kj_cmp = sb_nullstrcmp (R_last_ij, R_last_kj); + ij_ik_cmp = sb_nullstrcmp (R_last_ij, R_last_ik); + ik_kk_cmp = sb_nullstrcmp (R_last_ik, R_last_kk); + kk_kj_cmp = sb_nullstrcmp (R_last_kk, R_last_kj); + + /* Assign R_temp_(ik|kk|kj) to R_last[][] and remove epsilon as well + * as parentheses, so we can better compare the contents */ + + memset (&R_temp_ij, 0, sizeof (struct StringBuffer)); + memset (&R_temp_ik, 0, sizeof (struct StringBuffer)); + memset (&R_temp_kk, 0, sizeof (struct StringBuffer)); + memset (&R_temp_kj, 0, sizeof (struct StringBuffer)); + remove_epsilon (R_last_ik, &R_temp_ik); + remove_epsilon (R_last_kk, &R_temp_kk); + remove_epsilon (R_last_kj, &R_temp_kj); + remove_parentheses (&R_temp_ik); + remove_parentheses (&R_temp_kk); + remove_parentheses (&R_temp_kj); + clean_ik_kk_cmp = sb_nullstrcmp (R_last_ik, &R_temp_kk); + clean_kk_kj_cmp = sb_nullstrcmp (&R_temp_kk, R_last_kj); + + /* construct R_cur_l (and, if necessary R_cur_r) */ + if (GNUNET_YES != R_last_ij->null_flag) + { + /* Assign R_temp_ij to R_last_ij and remove epsilon as well + * as parentheses, so we can better compare the contents */ + remove_epsilon (R_last_ij, &R_temp_ij); + remove_parentheses (&R_temp_ij); + + if ( (0 == sb_strcmp (&R_temp_ij, &R_temp_ik)) && + (0 == sb_strcmp (&R_temp_ik, &R_temp_kk)) && + (0 == sb_strcmp (&R_temp_kk, &R_temp_kj)) ) + { + if (0 == R_temp_ij.slen) + { + R_cur_r->null_flag = GNUNET_NO; + } + else if ((0 == sb_strncmp_cstr (R_last_ij, "(|", 2)) || + (0 == sb_strncmp_cstr (R_last_ik, "(|", 2) && + 0 == sb_strncmp_cstr (R_last_kj, "(|", 2))) + { + /* + * a|(e|a)a*(e|a) = a* + * a|(e|a)(e|a)*(e|a) = a* + * (e|a)|aa*a = a* + * (e|a)|aa*(e|a) = a* + * (e|a)|(e|a)a*a = a* + * (e|a)|(e|a)a*(e|a) = a* + * (e|a)|(e|a)(e|a)*(e|a) = a* + */ + if (GNUNET_YES == needs_parentheses (&R_temp_ij)) + sb_printf1 (R_cur_r, "(%.*s)*", 3, &R_temp_ij); + else + sb_printf1 (R_cur_r, "%.*s*", 1, &R_temp_ij); + } + else + { + /* + * a|aa*a = a+ + * a|(e|a)a*a = a+ + * a|aa*(e|a) = a+ + * a|(e|a)(e|a)*a = a+ + * a|a(e|a)*(e|a) = a+ + */ + if (GNUNET_YES == needs_parentheses (&R_temp_ij)) + sb_printf1 (R_cur_r, "(%.*s)+", 3, &R_temp_ij); + else + sb_printf1 (R_cur_r, "%.*s+", 1, &R_temp_ij); + } + } + else if ( (0 == ij_ik_cmp) && (0 == clean_kk_kj_cmp) && (0 != clean_ik_kk_cmp) ) + { + /* a|ab*b = ab* */ + if (0 == R_last_kk->slen) + sb_strdup (R_cur_r, R_last_ij); + else if (GNUNET_YES == needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "%.*s(%.*s)*", 3, R_last_ij, &R_temp_kk); + else + sb_printf2 (R_cur_r, "%.*s%.*s*", 1, R_last_ij, R_last_kk); + R_cur_l->null_flag = GNUNET_YES; + } + else if ( (0 == ij_kj_cmp) && (0 == clean_ik_kk_cmp) && (0 != clean_kk_kj_cmp)) + { + /* a|bb*a = b*a */ + if (R_last_kk->slen < 1) + { + sb_strdup (R_cur_r, R_last_kj); + } + else if (GNUNET_YES == needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)*%.*s", 3, &R_temp_kk, R_last_kj); + else + sb_printf2 (R_cur_r, "%.*s*%.*s", 1, &R_temp_kk, R_last_kj); + + R_cur_l->null_flag = GNUNET_YES; + } + else if ( (0 == ij_ik_cmp) && (0 == kk_kj_cmp) && (! has_epsilon (R_last_ij)) && + has_epsilon (R_last_kk)) + { + /* a|a(e|b)*(e|b) = a|ab* = a|a|ab|abb|abbb|... = ab* */ + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "%.*s(%.*s)*", 3, R_last_ij, &R_temp_kk); + else + sb_printf2 (R_cur_r, "%.*s%.*s*", 1, R_last_ij, &R_temp_kk); + R_cur_l->null_flag = GNUNET_YES; + } + else if ( (0 == ij_kj_cmp) && (0 == ik_kk_cmp) && (! has_epsilon (R_last_ij)) && + has_epsilon (R_last_kk)) + { + /* a|(e|b)(e|b)*a = a|b*a = a|a|ba|bba|bbba|... = b*a */ + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)*%.*s", 3, &R_temp_kk, R_last_ij); + else + sb_printf2 (R_cur_r, "%.*s*%.*s", 1, &R_temp_kk, R_last_ij); + R_cur_l->null_flag = GNUNET_YES; + } + else + { + sb_strdup (R_cur_l, R_last_ij); + remove_parentheses (R_cur_l); + } + } + else + { + /* we have no left side */ + R_cur_l->null_flag = GNUNET_YES; + } + + /* construct R_cur_r, if not already constructed */ + if (GNUNET_YES == R_cur_r->null_flag) + { + length = R_temp_kk.slen - R_last_ik->slen; + + /* a(ba)*bx = (ab)+x */ + if ( (length > 0) && + (GNUNET_YES != R_last_kk->null_flag) && + (0 < R_last_kk->slen) && + (GNUNET_YES != R_last_kj->null_flag) && + (0 < R_last_kj->slen) && + (GNUNET_YES != R_last_ik->null_flag) && + (0 < R_last_ik->slen) && + (0 == sb_strkcmp (&R_temp_kk, R_last_ik, length)) && + (0 == sb_strncmp (&R_temp_kk, R_last_kj, length)) ) + { + struct StringBuffer temp_a; + struct StringBuffer temp_b; + + sb_init (&temp_a, length); + sb_init (&temp_b, R_last_kj->slen - length); + + length_l = length; + temp_a.sbuf = temp_a.abuf; + GNUNET_memcpy (temp_a.sbuf, R_last_kj->sbuf, length_l); + temp_a.slen = length_l; + + length_r = R_last_kj->slen - length; + temp_b.sbuf = temp_b.abuf; + GNUNET_memcpy (temp_b.sbuf, &R_last_kj->sbuf[length], length_r); + temp_b.slen = length_r; + + /* e|(ab)+ = (ab)* */ + if ( (GNUNET_YES != R_cur_l->null_flag) && + (0 == R_cur_l->slen) && + (0 == temp_b.slen) ) + { + sb_printf2 (R_cur_r, "(%.*s%.*s)*", 3, R_last_ik, &temp_a); + sb_free (R_cur_l); + R_cur_l->null_flag = GNUNET_YES; + } + else + { + sb_printf3 (R_cur_r, "(%.*s%.*s)+%.*s", 3, R_last_ik, &temp_a, &temp_b); + } + sb_free (&temp_a); + sb_free (&temp_b); + } + else if (0 == sb_strcmp (&R_temp_ik, &R_temp_kk) && + 0 == sb_strcmp (&R_temp_kk, &R_temp_kj)) + { + /* + * (e|a)a*(e|a) = a* + * (e|a)(e|a)*(e|a) = a* + */ + if (has_epsilon (R_last_ik) && has_epsilon (R_last_kj)) + { + if (needs_parentheses (&R_temp_kk)) + sb_printf1 (R_cur_r, "(%.*s)*", 3, &R_temp_kk); + else + sb_printf1 (R_cur_r, "%.*s*", 1, &R_temp_kk); + } + /* aa*a = a+a */ + else if ( (0 == clean_ik_kk_cmp) && + (0 == clean_kk_kj_cmp) && + (! has_epsilon (R_last_ik)) ) + { + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)+%.*s", 3, &R_temp_kk, &R_temp_kk); + else + sb_printf2 (R_cur_r, "%.*s+%.*s", 1, &R_temp_kk, &R_temp_kk); + } + /* + * (e|a)a*a = a+ + * aa*(e|a) = a+ + * a(e|a)*(e|a) = a+ + * (e|a)a*a = a+ + */ + else + { + eps_check = + (has_epsilon (R_last_ik) + has_epsilon (R_last_kk) + + has_epsilon (R_last_kj)); + + if (1 == eps_check) + { + if (needs_parentheses (&R_temp_kk)) + sb_printf1 (R_cur_r, "(%.*s)+", 3, &R_temp_kk); + else + sb_printf1 (R_cur_r, "%.*s+", 1, &R_temp_kk); + } + } + } + /* + * aa*b = a+b + * (e|a)(e|a)*b = a*b + */ + else if (0 == sb_strcmp (&R_temp_ik, &R_temp_kk)) + { + if (has_epsilon (R_last_ik)) + { + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)*%.*s", 3, &R_temp_kk, R_last_kj); + else + sb_printf2 (R_cur_r, "%.*s*%.*s", 1, &R_temp_kk, R_last_kj); + } + else + { + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)+%.*s", 3, &R_temp_kk, R_last_kj); + else + sb_printf2 (R_cur_r, "%.*s+%.*s", 1, &R_temp_kk, R_last_kj); + } + } + /* + * ba*a = ba+ + * b(e|a)*(e|a) = ba* + */ + else if (0 == sb_strcmp (&R_temp_kk, &R_temp_kj)) + { + if (has_epsilon (R_last_kj)) + { + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "%.*s(%.*s)*", 3, R_last_ik, &R_temp_kk); + else + sb_printf2 (R_cur_r, "%.*s%.*s*", 1, R_last_ik, &R_temp_kk); + } + else + { + if (needs_parentheses (&R_temp_kk)) + sb_printf2 (R_cur_r, "(%.*s)+%.*s", 3, R_last_ik, &R_temp_kk); + else + sb_printf2 (R_cur_r, "%.*s+%.*s", 1, R_last_ik, &R_temp_kk); + } + } + else + { + if (0 < R_temp_kk.slen) + { + if (needs_parentheses (&R_temp_kk)) + { + sb_printf3 (R_cur_r, "%.*s(%.*s)*%.*s", 3, R_last_ik, &R_temp_kk, + R_last_kj); + } + else + { + sb_printf3 (R_cur_r, "%.*s%.*s*%.*s", 1, R_last_ik, &R_temp_kk, + R_last_kj); + } + } + else + { + sb_printf2 (R_cur_r, "%.*s%.*s", 0, R_last_ik, R_last_kj); + } + } + } + sb_free (&R_temp_ij); + sb_free (&R_temp_ik); + sb_free (&R_temp_kk); + sb_free (&R_temp_kj); + + if ( (GNUNET_YES == R_cur_l->null_flag) && + (GNUNET_YES == R_cur_r->null_flag) ) + { + R_cur_ij->null_flag = GNUNET_YES; + return; + } + + if ( (GNUNET_YES != R_cur_l->null_flag) && + (GNUNET_YES == R_cur_r->null_flag) ) + { + struct StringBuffer tmp; + + tmp = *R_cur_ij; + *R_cur_ij = *R_cur_l; + *R_cur_l = tmp; + return; + } + + if ( (GNUNET_YES == R_cur_l->null_flag) && + (GNUNET_YES != R_cur_r->null_flag) ) + { + struct StringBuffer tmp; + + tmp = *R_cur_ij; + *R_cur_ij = *R_cur_r; + *R_cur_r = tmp; + return; + } + + if (0 == sb_nullstrcmp (R_cur_l, R_cur_r)) + { + struct StringBuffer tmp; + + tmp = *R_cur_ij; + *R_cur_ij = *R_cur_l; + *R_cur_l = tmp; + return; + } + sb_printf2 (R_cur_ij, "(%.*s|%.*s)", 3, R_cur_l, R_cur_r); +} + + +/** + * Create proofs for all states in the given automaton. Implementation of the + * algorithm descriped in chapter 3.2.1 of "Automata Theory, Languages, and + * Computation 3rd Edition" by Hopcroft, Motwani and Ullman. + * + * Each state in the automaton gets assigned 'proof' and 'hash' (hash of the + * proof) fields. The starting state will only have a valid proof/hash if it has + * any incoming transitions. + * + * @param a automaton for which to assign proofs and hashes, must not be NULL + */ +static int +automaton_create_proofs (struct REGEX_INTERNAL_Automaton *a) +{ + unsigned int n = a->state_count; + struct REGEX_INTERNAL_State *states[n]; + struct StringBuffer *R_last; + struct StringBuffer *R_cur; + struct StringBuffer R_cur_r; + struct StringBuffer R_cur_l; + struct StringBuffer *R_swap; + struct REGEX_INTERNAL_Transition *t; + struct StringBuffer complete_regex; + unsigned int i; + unsigned int j; + unsigned int k; + + R_last = GNUNET_malloc_large (sizeof (struct StringBuffer) * n * n); + R_cur = GNUNET_malloc_large (sizeof (struct StringBuffer) * n * n); + if ( (NULL == R_last) || + (NULL == R_cur) ) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc"); + GNUNET_free_non_null (R_cur); + GNUNET_free_non_null (R_last); + return GNUNET_SYSERR; + } + + /* create depth-first numbering of the states, initializes 'state' */ + REGEX_INTERNAL_automaton_traverse (a, a->start, NULL, NULL, &number_states, + states); + + for (i = 0; i < n; i++) + GNUNET_assert (NULL != states[i]); + for (i = 0; i < n; i++) + for (j = 0; j < n; j++) + R_last[i *n + j].null_flag = GNUNET_YES; + + /* Compute regular expressions of length "1" between each pair of states */ + for (i = 0; i < n; i++) + { + for (t = states[i]->transitions_head; NULL != t; t = t->next) + { + j = t->to_state->dfs_id; + if (GNUNET_YES == R_last[i * n + j].null_flag) + { + sb_strdup_cstr (&R_last[i * n + j], t->label); + } + else + { + sb_append_cstr (&R_last[i * n + j], "|"); + sb_append_cstr (&R_last[i * n + j], t->label); + } + } + /* add self-loop: i is reachable from i via epsilon-transition */ + if (GNUNET_YES == R_last[i * n + i].null_flag) + { + R_last[i * n + i].slen = 0; + R_last[i * n + i].null_flag = GNUNET_NO; + } + else + { + sb_wrap (&R_last[i * n + i], "(|%.*s)", 3); + } + } + for (i = 0; i < n; i++) + for (j = 0; j < n; j++) + if (needs_parentheses (&R_last[i * n + j])) + sb_wrap (&R_last[i * n + j], "(%.*s)", 2); + /* Compute regular expressions of length "k" between each pair of states per + * induction */ + memset (&R_cur_l, 0, sizeof (struct StringBuffer)); + memset (&R_cur_r, 0, sizeof (struct StringBuffer)); + for (k = 0; k < n; k++) + { + for (i = 0; i < n; i++) + { + for (j = 0; j < n; j++) + { + /* Basis for the recursion: + * $R^{(k)}_{ij} = R^{(k-1)}_{ij} | R^{(k-1)}_{ik} ( R^{(k-1)}_{kk} )^* R^{(k-1)}_{kj} + * R_last == R^{(k-1)}, R_cur == R^{(k)} + */ + + /* Create R_cur[i][j] and simplify the expression */ + automaton_create_proofs_simplify (&R_last[i * n + j], &R_last[i * n + k], + &R_last[k * n + k], &R_last[k * n + j], + &R_cur[i * n + j], + &R_cur_l, &R_cur_r); + } + } + /* set R_last = R_cur */ + R_swap = R_last; + R_last = R_cur; + R_cur = R_swap; + /* clear 'R_cur' for next iteration */ + for (i = 0; i < n; i++) + for (j = 0; j < n; j++) + R_cur[i * n + j].null_flag = GNUNET_YES; + } + sb_free (&R_cur_l); + sb_free (&R_cur_r); + /* assign proofs and hashes */ + for (i = 0; i < n; i++) + { + if (GNUNET_YES != R_last[a->start->dfs_id * n + i].null_flag) + { + states[i]->proof = GNUNET_strndup (R_last[a->start->dfs_id * n + i].sbuf, + R_last[a->start->dfs_id * n + i].slen); + GNUNET_CRYPTO_hash (states[i]->proof, strlen (states[i]->proof), + &states[i]->hash); + } + } + + /* complete regex for whole DFA: union of all pairs (start state/accepting + * state(s)). */ + sb_init (&complete_regex, 16 * n); + for (i = 0; i < n; i++) + { + if (states[i]->accepting) + { + if ( (0 == complete_regex.slen) && + (0 < R_last[a->start->dfs_id * n + i].slen) ) + { + sb_append (&complete_regex, + &R_last[a->start->dfs_id * n + i]); + } + else if ( (GNUNET_YES != R_last[a->start->dfs_id * n + i].null_flag) && + (0 < R_last[a->start->dfs_id * n + i].slen) ) + { + sb_append_cstr (&complete_regex, "|"); + sb_append (&complete_regex, + &R_last[a->start->dfs_id * n + i]); + } + } + } + a->canonical_regex = GNUNET_strndup (complete_regex.sbuf, complete_regex.slen); + + /* cleanup */ + sb_free (&complete_regex); + for (i = 0; i < n; i++) + for (j = 0; j < n; j++) + { + sb_free (&R_cur[i * n + j]); + sb_free (&R_last[i * n + j]); + } + GNUNET_free (R_cur); + GNUNET_free (R_last); + return GNUNET_OK; +} + + +/** + * Creates a new DFA state based on a set of NFA states. Needs to be freed using + * automaton_destroy_state. + * + * @param ctx context + * @param nfa_states set of NFA states on which the DFA should be based on + * + * @return new DFA state + */ +static struct REGEX_INTERNAL_State * +dfa_state_create (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_StateSet *nfa_states) +{ + struct REGEX_INTERNAL_State *s; + char *pos; + size_t len; + struct REGEX_INTERNAL_State *cstate; + struct REGEX_INTERNAL_Transition *ctran; + unsigned int i; + + s = GNUNET_new (struct REGEX_INTERNAL_State); + s->id = ctx->state_id++; + s->index = -1; + s->lowlink = -1; + + if (NULL == nfa_states) + { + GNUNET_asprintf (&s->name, "s%i", s->id); + return s; + } + + s->nfa_set = *nfa_states; + + if (nfa_states->off < 1) + return s; + + /* Create a name based on 'nfa_states' */ + len = nfa_states->off * 14 + 4; + s->name = GNUNET_malloc (len); + strcat (s->name, "{"); + pos = s->name + 1; + + for (i = 0; i < nfa_states->off; i++) + { + cstate = nfa_states->states[i]; + GNUNET_snprintf (pos, + pos - s->name + len, + "%i,", + cstate->id); + pos += strlen (pos); + + /* Add a transition for each distinct label to NULL state */ + for (ctran = cstate->transitions_head; NULL != ctran; ctran = ctran->next) + if (NULL != ctran->label) + state_add_transition (ctx, s, ctran->label, NULL); + + /* If the nfa_states contain an accepting state, the new dfa state is also + * accepting. */ + if (cstate->accepting) + s->accepting = 1; + } + pos[-1] = '}'; + s->name = GNUNET_realloc (s->name, strlen (s->name) + 1); + + memset (nfa_states, 0, sizeof (struct REGEX_INTERNAL_StateSet)); + return s; +} + + +/** + * Move from the given state 's' to the next state on transition 'str'. Consumes + * as much of the given 'str' as possible (usefull for strided DFAs). On return + * 's' will point to the next state, and the length of the substring used for + * this transition will be returned. If no transition possible 0 is returned and + * 's' points to NULL. + * + * @param s starting state, will point to the next state or NULL (if no + * transition possible) + * @param str edge label to follow (will match longest common prefix) + * + * @return length of the substring comsumed from 'str' + */ +static unsigned int +dfa_move (struct REGEX_INTERNAL_State **s, const char *str) +{ + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_State *new_s; + unsigned int len; + unsigned int max_len; + + if (NULL == s) + return 0; + + new_s = NULL; + max_len = 0; + for (t = (*s)->transitions_head; NULL != t; t = t->next) + { + len = strlen (t->label); + + if (0 == strncmp (t->label, str, len)) + { + if (len >= max_len) + { + max_len = len; + new_s = t->to_state; + } + } + } + + *s = new_s; + return max_len; +} + + +/** + * Set the given state 'marked' to #GNUNET_YES. Used by the + * #dfa_remove_unreachable_states() function to detect unreachable states in the + * automaton. + * + * @param cls closure, not used. + * @param count count, not used. + * @param s state where the marked attribute will be set to #GNUNET_YES. + */ +static void +mark_states (void *cls, + const unsigned int count, + struct REGEX_INTERNAL_State *s) +{ + s->marked = GNUNET_YES; +} + + +/** + * Remove all unreachable states from DFA 'a'. Unreachable states are those + * states that are not reachable from the starting state. + * + * @param a DFA automaton + */ +static void +dfa_remove_unreachable_states (struct REGEX_INTERNAL_Automaton *a) +{ + struct REGEX_INTERNAL_State *s; + struct REGEX_INTERNAL_State *s_next; + + /* 1. unmark all states */ + for (s = a->states_head; NULL != s; s = s->next) + s->marked = GNUNET_NO; + + /* 2. traverse dfa from start state and mark all visited states */ + REGEX_INTERNAL_automaton_traverse (a, a->start, NULL, NULL, &mark_states, NULL); + + /* 3. delete all states that were not visited */ + for (s = a->states_head; NULL != s; s = s_next) + { + s_next = s->next; + if (GNUNET_NO == s->marked) + automaton_remove_state (a, s); + } +} + + +/** + * Remove all dead states from the DFA 'a'. Dead states are those states that do + * not transition to any other state but themselves. + * + * @param a DFA automaton + */ +static void +dfa_remove_dead_states (struct REGEX_INTERNAL_Automaton *a) +{ + struct REGEX_INTERNAL_State *s; + struct REGEX_INTERNAL_State *s_next; + struct REGEX_INTERNAL_Transition *t; + int dead; + + GNUNET_assert (DFA == a->type); + + for (s = a->states_head; NULL != s; s = s_next) + { + s_next = s->next; + + if (s->accepting) + continue; + + dead = 1; + for (t = s->transitions_head; NULL != t; t = t->next) + { + if (NULL != t->to_state && t->to_state != s) + { + dead = 0; + break; + } + } + + if (0 == dead) + continue; + + /* state s is dead, remove it */ + automaton_remove_state (a, s); + } +} + + +/** + * Merge all non distinguishable states in the DFA 'a' + * + * @param ctx context + * @param a DFA automaton + * @return #GNUNET_OK on success + */ +static int +dfa_merge_nondistinguishable_states (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_Automaton *a) +{ + uint32_t *table; + struct REGEX_INTERNAL_State *s1; + struct REGEX_INTERNAL_State *s2; + struct REGEX_INTERNAL_Transition *t1; + struct REGEX_INTERNAL_Transition *t2; + struct REGEX_INTERNAL_State *s1_next; + struct REGEX_INTERNAL_State *s2_next; + int change; + unsigned int num_equal_edges; + unsigned int i; + unsigned int state_cnt; + unsigned long long idx; + unsigned long long idx1; + + if ( (NULL == a) || (0 == a->state_count) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not merge nondistinguishable states, automaton was NULL.\n"); + return GNUNET_SYSERR; + } + + state_cnt = a->state_count; + table = GNUNET_malloc_large ((sizeof (uint32_t) * state_cnt * state_cnt / 32) + sizeof (uint32_t)); + if (NULL == table) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc"); + return GNUNET_SYSERR; + } + + for (i = 0, s1 = a->states_head; NULL != s1; s1 = s1->next) + s1->marked = i++; + + /* Mark all pairs of accepting/!accepting states */ + for (s1 = a->states_head; NULL != s1; s1 = s1->next) + for (s2 = a->states_head; NULL != s2; s2 = s2->next) + if ( (s1->accepting && !s2->accepting) || + (!s1->accepting && s2->accepting) ) + { + idx = (unsigned long long) s1->marked * state_cnt + s2->marked; + table[idx / 32] |= (1U << (idx % 32)); + } + + /* Find all equal states */ + change = 1; + while (0 != change) + { + change = 0; + for (s1 = a->states_head; NULL != s1; s1 = s1->next) + { + for (s2 = a->states_head; NULL != s2 && s1 != s2; s2 = s2->next) + { + idx = (unsigned long long) s1->marked * state_cnt + s2->marked; + if (0 != (table[idx / 32] & (1U << (idx % 32)))) + continue; + num_equal_edges = 0; + for (t1 = s1->transitions_head; NULL != t1; t1 = t1->next) + { + for (t2 = s2->transitions_head; NULL != t2; t2 = t2->next) + { + if (0 == strcmp (t1->label, t2->label)) + { + num_equal_edges++; + /* same edge, but targets definitively different, so we're different + as well */ + if (t1->to_state->marked > t2->to_state->marked) + idx1 = (unsigned long long) t1->to_state->marked * state_cnt + t2->to_state->marked; + else + idx1 = (unsigned long long) t2->to_state->marked * state_cnt + t1->to_state->marked; + if (0 != (table[idx1 / 32] & (1U << (idx1 % 32)))) + { + table[idx / 32] |= (1U << (idx % 32)); + change = 1; /* changed a marker, need to run again */ + } + } + } + } + if ( (num_equal_edges != s1->transition_count) || + (num_equal_edges != s2->transition_count) ) + { + /* Make sure ALL edges of possible equal states are the same */ + table[idx / 32] |= (1U << (idx % 32)); + change = 1; /* changed a marker, need to run again */ + } + } + } + } + + /* Merge states that are equal */ + for (s1 = a->states_head; NULL != s1; s1 = s1_next) + { + s1_next = s1->next; + for (s2 = a->states_head; NULL != s2 && s1 != s2; s2 = s2_next) + { + s2_next = s2->next; + idx = (unsigned long long) s1->marked * state_cnt + s2->marked; + if (0 == (table[idx / 32] & (1U << (idx % 32)))) + automaton_merge_states (ctx, a, s1, s2); + } + } + + GNUNET_free (table); + return GNUNET_OK; +} + + +/** + * Minimize the given DFA 'a' by removing all unreachable states, removing all + * dead states and merging all non distinguishable states + * + * @param ctx context + * @param a DFA automaton + * @return GNUNET_OK on success + */ +static int +dfa_minimize (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_Automaton *a) +{ + if (NULL == a) + return GNUNET_SYSERR; + + GNUNET_assert (DFA == a->type); + + /* 1. remove unreachable states */ + dfa_remove_unreachable_states (a); + + /* 2. remove dead states */ + dfa_remove_dead_states (a); + + /* 3. Merge nondistinguishable states */ + if (GNUNET_OK != dfa_merge_nondistinguishable_states (ctx, a)) + return GNUNET_SYSERR; + return GNUNET_OK; +} + + +/** + * Context for adding strided transitions to a DFA. + */ +struct REGEX_INTERNAL_Strided_Context +{ + /** + * Length of the strides. + */ + const unsigned int stride; + + /** + * Strided transitions DLL. New strided transitions will be stored in this DLL + * and afterwards added to the DFA. + */ + struct REGEX_INTERNAL_Transition *transitions_head; + + /** + * Strided transitions DLL. + */ + struct REGEX_INTERNAL_Transition *transitions_tail; +}; + + +/** + * Recursive helper function to add strides to a DFA. + * + * @param cls context, contains stride length and strided transitions DLL. + * @param depth current depth of the depth-first traversal of the graph. + * @param label current label, string that contains all labels on the path from + * 'start' to 's'. + * @param start start state for the depth-first traversal of the graph. + * @param s current state in the depth-first traversal + */ +static void +dfa_add_multi_strides_helper (void *cls, const unsigned int depth, char *label, + struct REGEX_INTERNAL_State *start, + struct REGEX_INTERNAL_State *s) +{ + struct REGEX_INTERNAL_Strided_Context *ctx = cls; + struct REGEX_INTERNAL_Transition *t; + char *new_label; + + if (depth == ctx->stride) + { + t = GNUNET_new (struct REGEX_INTERNAL_Transition); + t->label = GNUNET_strdup (label); + t->to_state = s; + t->from_state = start; + GNUNET_CONTAINER_DLL_insert (ctx->transitions_head, ctx->transitions_tail, + t); + } + else + { + for (t = s->transitions_head; NULL != t; t = t->next) + { + /* Do not consider self-loops, because it end's up in too many + * transitions */ + if (t->to_state == t->from_state) + continue; + + if (NULL != label) + { + GNUNET_asprintf (&new_label, "%s%s", label, t->label); + } + else + new_label = GNUNET_strdup (t->label); + + dfa_add_multi_strides_helper (cls, (depth + 1), new_label, start, + t->to_state); + } + } + GNUNET_free_non_null (label); +} + + +/** + * Function called for each state in the DFA. Starts a traversal of depth set in + * context starting from state 's'. + * + * @param cls context. + * @param count not used. + * @param s current state. + */ +static void +dfa_add_multi_strides (void *cls, const unsigned int count, + struct REGEX_INTERNAL_State *s) +{ + dfa_add_multi_strides_helper (cls, 0, NULL, s, s); +} + + +/** + * Adds multi-strided transitions to the given 'dfa'. + * + * @param regex_ctx regex context needed to add transitions to the automaton. + * @param dfa DFA to which the multi strided transitions should be added. + * @param stride_len length of the strides. + */ +void +REGEX_INTERNAL_dfa_add_multi_strides (struct REGEX_INTERNAL_Context *regex_ctx, + struct REGEX_INTERNAL_Automaton *dfa, + const unsigned int stride_len) +{ + struct REGEX_INTERNAL_Strided_Context ctx = { stride_len, NULL, NULL }; + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_Transition *t_next; + + if (1 > stride_len || GNUNET_YES == dfa->is_multistrided) + return; + + /* Compute the new transitions of given stride_len */ + REGEX_INTERNAL_automaton_traverse (dfa, dfa->start, NULL, NULL, + &dfa_add_multi_strides, &ctx); + + /* Add all the new transitions to the automaton. */ + for (t = ctx.transitions_head; NULL != t; t = t_next) + { + t_next = t->next; + state_add_transition (regex_ctx, t->from_state, t->label, t->to_state); + GNUNET_CONTAINER_DLL_remove (ctx.transitions_head, ctx.transitions_tail, t); + GNUNET_free_non_null (t->label); + GNUNET_free (t); + } + + /* Mark this automaton as multistrided */ + dfa->is_multistrided = GNUNET_YES; +} + +/** + * Recursive Helper function for DFA path compression. Does DFS on the DFA graph + * and adds new transitions to the given transitions DLL and marks states that + * should be removed by setting state->contained to GNUNET_YES. + * + * @param dfa DFA for which the paths should be compressed. + * @param start starting state for linear path search. + * @param cur current state in the recursive DFS. + * @param label current label (string of traversed labels). + * @param max_len maximal path compression length. + * @param transitions_head transitions DLL. + * @param transitions_tail transitions DLL. + */ +void +dfa_compress_paths_helper (struct REGEX_INTERNAL_Automaton *dfa, + struct REGEX_INTERNAL_State *start, + struct REGEX_INTERNAL_State *cur, char *label, + unsigned int max_len, + struct REGEX_INTERNAL_Transition **transitions_head, + struct REGEX_INTERNAL_Transition **transitions_tail) +{ + struct REGEX_INTERNAL_Transition *t; + char *new_label; + + + if (NULL != label && + ((cur->incoming_transition_count > 1 || GNUNET_YES == cur->accepting || + GNUNET_YES == cur->marked) || (start != dfa->start && max_len > 0 && + max_len == strlen (label)) || + (start == dfa->start && GNUNET_REGEX_INITIAL_BYTES == strlen (label)))) + { + t = GNUNET_new (struct REGEX_INTERNAL_Transition); + t->label = GNUNET_strdup (label); + t->to_state = cur; + t->from_state = start; + GNUNET_CONTAINER_DLL_insert (*transitions_head, *transitions_tail, t); + + if (GNUNET_NO == cur->marked) + { + dfa_compress_paths_helper (dfa, cur, cur, NULL, max_len, transitions_head, + transitions_tail); + } + return; + } + else if (cur != start) + cur->contained = GNUNET_YES; + + if (GNUNET_YES == cur->marked && cur != start) + return; + + cur->marked = GNUNET_YES; + + + for (t = cur->transitions_head; NULL != t; t = t->next) + { + if (NULL != label) + GNUNET_asprintf (&new_label, "%s%s", label, t->label); + else + new_label = GNUNET_strdup (t->label); + + if (t->to_state != cur) + { + dfa_compress_paths_helper (dfa, start, t->to_state, new_label, max_len, + transitions_head, transitions_tail); + } + GNUNET_free (new_label); + } +} + + +/** + * Compress paths in the given 'dfa'. Linear paths like 0->1->2->3 will be + * compressed to 0->3 by combining transitions. + * + * @param regex_ctx context for adding new transitions. + * @param dfa DFA representation, will directly modify the given DFA. + * @param max_len maximal length of the compressed paths. + */ +static void +dfa_compress_paths (struct REGEX_INTERNAL_Context *regex_ctx, + struct REGEX_INTERNAL_Automaton *dfa, unsigned int max_len) +{ + struct REGEX_INTERNAL_State *s; + struct REGEX_INTERNAL_State *s_next; + struct REGEX_INTERNAL_Transition *t; + struct REGEX_INTERNAL_Transition *t_next; + struct REGEX_INTERNAL_Transition *transitions_head = NULL; + struct REGEX_INTERNAL_Transition *transitions_tail = NULL; + + if (NULL == dfa) + return; + + /* Count the incoming transitions on each state. */ + for (s = dfa->states_head; NULL != s; s = s->next) + { + for (t = s->transitions_head; NULL != t; t = t->next) + { + if (NULL != t->to_state) + t->to_state->incoming_transition_count++; + } + } + + /* Unmark all states. */ + for (s = dfa->states_head; NULL != s; s = s->next) + { + s->marked = GNUNET_NO; + s->contained = GNUNET_NO; + } + + /* Add strides and mark states that can be deleted. */ + dfa_compress_paths_helper (dfa, dfa->start, dfa->start, NULL, max_len, + &transitions_head, &transitions_tail); + + /* Add all the new transitions to the automaton. */ + for (t = transitions_head; NULL != t; t = t_next) + { + t_next = t->next; + state_add_transition (regex_ctx, t->from_state, t->label, t->to_state); + GNUNET_CONTAINER_DLL_remove (transitions_head, transitions_tail, t); + GNUNET_free_non_null (t->label); + GNUNET_free (t); + } + + /* Remove marked states (including their incoming and outgoing transitions). */ + for (s = dfa->states_head; NULL != s; s = s_next) + { + s_next = s->next; + if (GNUNET_YES == s->contained) + automaton_remove_state (dfa, s); + } +} + + +/** + * Creates a new NFA fragment. Needs to be cleared using + * automaton_fragment_clear. + * + * @param start starting state + * @param end end state + * + * @return new NFA fragment + */ +static struct REGEX_INTERNAL_Automaton * +nfa_fragment_create (struct REGEX_INTERNAL_State *start, + struct REGEX_INTERNAL_State *end) +{ + struct REGEX_INTERNAL_Automaton *n; + + n = GNUNET_new (struct REGEX_INTERNAL_Automaton); + + n->type = NFA; + n->start = NULL; + n->end = NULL; + n->state_count = 0; + + if (NULL == start || NULL == end) + return n; + + automaton_add_state (n, end); + automaton_add_state (n, start); + + n->state_count = 2; + + n->start = start; + n->end = end; + + return n; +} + + +/** + * Adds a list of states to the given automaton 'n'. + * + * @param n automaton to which the states should be added + * @param states_head head of the DLL of states + * @param states_tail tail of the DLL of states + */ +static void +nfa_add_states (struct REGEX_INTERNAL_Automaton *n, + struct REGEX_INTERNAL_State *states_head, + struct REGEX_INTERNAL_State *states_tail) +{ + struct REGEX_INTERNAL_State *s; + + if (NULL == n || NULL == states_head) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not add states\n"); + return; + } + + if (NULL == n->states_head) + { + n->states_head = states_head; + n->states_tail = states_tail; + return; + } + + if (NULL != states_head) + { + n->states_tail->next = states_head; + n->states_tail = states_tail; + } + + for (s = states_head; NULL != s; s = s->next) + n->state_count++; +} + + +/** + * Creates a new NFA state. Needs to be freed using automaton_destroy_state. + * + * @param ctx context + * @param accepting is it an accepting state or not + * + * @return new NFA state + */ +static struct REGEX_INTERNAL_State * +nfa_state_create (struct REGEX_INTERNAL_Context *ctx, int accepting) +{ + struct REGEX_INTERNAL_State *s; + + s = GNUNET_new (struct REGEX_INTERNAL_State); + s->id = ctx->state_id++; + s->accepting = accepting; + s->marked = GNUNET_NO; + s->contained = 0; + s->index = -1; + s->lowlink = -1; + s->scc_id = 0; + s->name = NULL; + GNUNET_asprintf (&s->name, "s%i", s->id); + + return s; +} + + +/** + * Calculates the closure set for the given set of states. + * + * @param ret set to sorted nfa closure on 'label' (epsilon closure if 'label' is NULL) + * @param nfa the NFA containing 's' + * @param states list of states on which to base the closure on + * @param label transitioning label for which to base the closure on, + * pass NULL for epsilon transition + */ +static void +nfa_closure_set_create (struct REGEX_INTERNAL_StateSet *ret, + struct REGEX_INTERNAL_Automaton *nfa, + struct REGEX_INTERNAL_StateSet *states, const char *label) +{ + struct REGEX_INTERNAL_State *s; + unsigned int i; + struct REGEX_INTERNAL_StateSet_MDLL cls_stack; + struct REGEX_INTERNAL_State *clsstate; + struct REGEX_INTERNAL_State *currentstate; + struct REGEX_INTERNAL_Transition *ctran; + + memset (ret, 0, sizeof (struct REGEX_INTERNAL_StateSet)); + if (NULL == states) + return; + + for (i = 0; i < states->off; i++) + { + s = states->states[i]; + + /* Add start state to closure only for epsilon closure */ + if (NULL == label) + state_set_append (ret, s); + + /* initialize work stack */ + cls_stack.head = NULL; + cls_stack.tail = NULL; + GNUNET_CONTAINER_MDLL_insert (ST, cls_stack.head, cls_stack.tail, s); + cls_stack.len = 1; + + while (NULL != (currentstate = cls_stack.tail)) + { + GNUNET_CONTAINER_MDLL_remove (ST, cls_stack.head, cls_stack.tail, + currentstate); + cls_stack.len--; + for (ctran = currentstate->transitions_head; NULL != ctran; + ctran = ctran->next) + { + if (NULL == (clsstate = ctran->to_state)) + continue; + if (0 != clsstate->contained) + continue; + if (0 != nullstrcmp (label, ctran->label)) + continue; + state_set_append (ret, clsstate); + GNUNET_CONTAINER_MDLL_insert_tail (ST, cls_stack.head, cls_stack.tail, + clsstate); + cls_stack.len++; + clsstate->contained = 1; + } + } + } + for (i = 0; i < ret->off; i++) + ret->states[i]->contained = 0; + + if (ret->off > 1) + qsort (ret->states, ret->off, sizeof (struct REGEX_INTERNAL_State *), + &state_compare); +} + + +/** + * Pops two NFA fragments (a, b) from the stack and concatenates them (ab) + * + * @param ctx context + */ +static void +nfa_add_concatenation (struct REGEX_INTERNAL_Context *ctx) +{ + struct REGEX_INTERNAL_Automaton *a; + struct REGEX_INTERNAL_Automaton *b; + struct REGEX_INTERNAL_Automaton *new_nfa; + + b = ctx->stack_tail; + GNUNET_assert (NULL != b); + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, b); + a = ctx->stack_tail; + GNUNET_assert (NULL != a); + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, a); + + state_add_transition (ctx, a->end, NULL, b->start); + a->end->accepting = 0; + b->end->accepting = 1; + + new_nfa = nfa_fragment_create (NULL, NULL); + nfa_add_states (new_nfa, a->states_head, a->states_tail); + nfa_add_states (new_nfa, b->states_head, b->states_tail); + new_nfa->start = a->start; + new_nfa->end = b->end; + new_nfa->state_count += a->state_count + b->state_count; + automaton_fragment_clear (a); + automaton_fragment_clear (b); + + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, new_nfa); +} + + +/** + * Pops a NFA fragment from the stack (a) and adds a new fragment (a*) + * + * @param ctx context + */ +static void +nfa_add_star_op (struct REGEX_INTERNAL_Context *ctx) +{ + struct REGEX_INTERNAL_Automaton *a; + struct REGEX_INTERNAL_Automaton *new_nfa; + struct REGEX_INTERNAL_State *start; + struct REGEX_INTERNAL_State *end; + + a = ctx->stack_tail; + + if (NULL == a) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "nfa_add_star_op failed, because there was no element on the stack"); + return; + } + + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, a); + + start = nfa_state_create (ctx, 0); + end = nfa_state_create (ctx, 1); + + state_add_transition (ctx, start, NULL, a->start); + state_add_transition (ctx, start, NULL, end); + state_add_transition (ctx, a->end, NULL, a->start); + state_add_transition (ctx, a->end, NULL, end); + + a->end->accepting = 0; + end->accepting = 1; + + new_nfa = nfa_fragment_create (start, end); + nfa_add_states (new_nfa, a->states_head, a->states_tail); + automaton_fragment_clear (a); + + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, new_nfa); +} + + +/** + * Pops an NFA fragment (a) from the stack and adds a new fragment (a+) + * + * @param ctx context + */ +static void +nfa_add_plus_op (struct REGEX_INTERNAL_Context *ctx) +{ + struct REGEX_INTERNAL_Automaton *a; + + a = ctx->stack_tail; + + if (NULL == a) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "nfa_add_plus_op failed, because there was no element on the stack"); + return; + } + + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, a); + + state_add_transition (ctx, a->end, NULL, a->start); + + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, a); +} + + +/** + * Pops an NFA fragment (a) from the stack and adds a new fragment (a?) + * + * @param ctx context + */ +static void +nfa_add_question_op (struct REGEX_INTERNAL_Context *ctx) +{ + struct REGEX_INTERNAL_Automaton *a; + struct REGEX_INTERNAL_Automaton *new_nfa; + struct REGEX_INTERNAL_State *start; + struct REGEX_INTERNAL_State *end; + + a = ctx->stack_tail; + if (NULL == a) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "nfa_add_question_op failed, because there was no element on the stack"); + return; + } + + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, a); + + start = nfa_state_create (ctx, 0); + end = nfa_state_create (ctx, 1); + + state_add_transition (ctx, start, NULL, a->start); + state_add_transition (ctx, start, NULL, end); + state_add_transition (ctx, a->end, NULL, end); + + a->end->accepting = 0; + + new_nfa = nfa_fragment_create (start, end); + nfa_add_states (new_nfa, a->states_head, a->states_tail); + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, new_nfa); + automaton_fragment_clear (a); +} + + +/** + * Pops two NFA fragments (a, b) from the stack and adds a new NFA fragment that + * alternates between a and b (a|b) + * + * @param ctx context + */ +static void +nfa_add_alternation (struct REGEX_INTERNAL_Context *ctx) +{ + struct REGEX_INTERNAL_Automaton *a; + struct REGEX_INTERNAL_Automaton *b; + struct REGEX_INTERNAL_Automaton *new_nfa; + struct REGEX_INTERNAL_State *start; + struct REGEX_INTERNAL_State *end; + + b = ctx->stack_tail; + GNUNET_assert (NULL != b); + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, b); + a = ctx->stack_tail; + GNUNET_assert (NULL != a); + GNUNET_CONTAINER_DLL_remove (ctx->stack_head, ctx->stack_tail, a); + + start = nfa_state_create (ctx, 0); + end = nfa_state_create (ctx, 1); + state_add_transition (ctx, start, NULL, a->start); + state_add_transition (ctx, start, NULL, b->start); + + state_add_transition (ctx, a->end, NULL, end); + state_add_transition (ctx, b->end, NULL, end); + + a->end->accepting = 0; + b->end->accepting = 0; + end->accepting = 1; + + new_nfa = nfa_fragment_create (start, end); + nfa_add_states (new_nfa, a->states_head, a->states_tail); + nfa_add_states (new_nfa, b->states_head, b->states_tail); + automaton_fragment_clear (a); + automaton_fragment_clear (b); + + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, new_nfa); +} + + +/** + * Adds a new nfa fragment to the stack + * + * @param ctx context + * @param label label for nfa transition + */ +static void +nfa_add_label (struct REGEX_INTERNAL_Context *ctx, const char *label) +{ + struct REGEX_INTERNAL_Automaton *n; + struct REGEX_INTERNAL_State *start; + struct REGEX_INTERNAL_State *end; + + GNUNET_assert (NULL != ctx); + + start = nfa_state_create (ctx, 0); + end = nfa_state_create (ctx, 1); + state_add_transition (ctx, start, label, end); + n = nfa_fragment_create (start, end); + GNUNET_assert (NULL != n); + GNUNET_CONTAINER_DLL_insert_tail (ctx->stack_head, ctx->stack_tail, n); +} + + +/** + * Initialize a new context + * + * @param ctx context + */ +static void +REGEX_INTERNAL_context_init (struct REGEX_INTERNAL_Context *ctx) +{ + if (NULL == ctx) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Context was NULL!"); + return; + } + ctx->state_id = 0; + ctx->transition_id = 0; + ctx->stack_head = NULL; + ctx->stack_tail = NULL; +} + + +/** + * Construct an NFA by parsing the regex string of length 'len'. + * + * @param regex regular expression string + * @param len length of the string + * + * @return NFA, needs to be freed using REGEX_INTERNAL_destroy_automaton + */ +struct REGEX_INTERNAL_Automaton * +REGEX_INTERNAL_construct_nfa (const char *regex, const size_t len) +{ + struct REGEX_INTERNAL_Context ctx; + struct REGEX_INTERNAL_Automaton *nfa; + const char *regexp; + char curlabel[2]; + char *error_msg; + unsigned int count; + unsigned int altcount; + unsigned int atomcount; + unsigned int poff; + unsigned int psize; + struct + { + int altcount; + int atomcount; + } *p; + + if (NULL == regex || 0 == strlen (regex) || 0 == len) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not parse regex. Empty regex string provided.\n"); + + return NULL; + } + REGEX_INTERNAL_context_init (&ctx); + + regexp = regex; + curlabel[1] = '\0'; + p = NULL; + error_msg = NULL; + altcount = 0; + atomcount = 0; + poff = 0; + psize = 0; + + for (count = 0; count < len && *regexp; count++, regexp++) + { + switch (*regexp) + { + case '(': + if (atomcount > 1) + { + --atomcount; + nfa_add_concatenation (&ctx); + } + if (poff == psize) + GNUNET_array_grow (p, psize, psize * 2 + 4); /* FIXME why *2 +4? */ + p[poff].altcount = altcount; + p[poff].atomcount = atomcount; + poff++; + altcount = 0; + atomcount = 0; + break; + case '|': + if (0 == atomcount) + { + error_msg = "Cannot append '|' to nothing"; + goto error; + } + while (--atomcount > 0) + nfa_add_concatenation (&ctx); + altcount++; + break; + case ')': + if (0 == poff) + { + error_msg = "Missing opening '('"; + goto error; + } + if (0 == atomcount) + { + /* Ignore this: "()" */ + poff--; + altcount = p[poff].altcount; + atomcount = p[poff].atomcount; + break; + } + while (--atomcount > 0) + nfa_add_concatenation (&ctx); + for (; altcount > 0; altcount--) + nfa_add_alternation (&ctx); + poff--; + altcount = p[poff].altcount; + atomcount = p[poff].atomcount; + atomcount++; + break; + case '*': + if (atomcount == 0) + { + error_msg = "Cannot append '*' to nothing"; + goto error; + } + nfa_add_star_op (&ctx); + break; + case '+': + if (atomcount == 0) + { + error_msg = "Cannot append '+' to nothing"; + goto error; + } + nfa_add_plus_op (&ctx); + break; + case '?': + if (atomcount == 0) + { + error_msg = "Cannot append '?' to nothing"; + goto error; + } + nfa_add_question_op (&ctx); + break; + default: + if (atomcount > 1) + { + --atomcount; + nfa_add_concatenation (&ctx); + } + curlabel[0] = *regexp; + nfa_add_label (&ctx, curlabel); + atomcount++; + break; + } + } + if (0 != poff) + { + error_msg = "Unbalanced parenthesis"; + goto error; + } + while (--atomcount > 0) + nfa_add_concatenation (&ctx); + for (; altcount > 0; altcount--) + nfa_add_alternation (&ctx); + + GNUNET_array_grow (p, psize, 0); + + nfa = ctx.stack_tail; + GNUNET_CONTAINER_DLL_remove (ctx.stack_head, ctx.stack_tail, nfa); + + if (NULL != ctx.stack_head) + { + error_msg = "Creating the NFA failed. NFA stack was not empty!"; + goto error; + } + + /* Remember the regex that was used to generate this NFA */ + nfa->regex = GNUNET_strdup (regex); + + /* create depth-first numbering of the states for pretty printing */ + REGEX_INTERNAL_automaton_traverse (nfa, NULL, NULL, NULL, &number_states, NULL); + + /* No multistriding added so far */ + nfa->is_multistrided = GNUNET_NO; + + return nfa; + +error: + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not parse regex: `%s'\n", regex); + if (NULL != error_msg) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", error_msg); + + GNUNET_free_non_null (p); + + while (NULL != (nfa = ctx.stack_head)) + { + GNUNET_CONTAINER_DLL_remove (ctx.stack_head, ctx.stack_tail, nfa); + REGEX_INTERNAL_automaton_destroy (nfa); + } + + return NULL; +} + + +/** + * Create DFA states based on given 'nfa' and starting with 'dfa_state'. + * + * @param ctx context. + * @param nfa NFA automaton. + * @param dfa DFA automaton. + * @param dfa_state current dfa state, pass epsilon closure of first nfa state + * for starting. + */ +static void +construct_dfa_states (struct REGEX_INTERNAL_Context *ctx, + struct REGEX_INTERNAL_Automaton *nfa, + struct REGEX_INTERNAL_Automaton *dfa, + struct REGEX_INTERNAL_State *dfa_state) +{ + struct REGEX_INTERNAL_Transition *ctran; + struct REGEX_INTERNAL_State *new_dfa_state; + struct REGEX_INTERNAL_State *state_contains; + struct REGEX_INTERNAL_State *state_iter; + struct REGEX_INTERNAL_StateSet tmp; + struct REGEX_INTERNAL_StateSet nfa_set; + + for (ctran = dfa_state->transitions_head; NULL != ctran; ctran = ctran->next) + { + if (NULL == ctran->label || NULL != ctran->to_state) + continue; + + nfa_closure_set_create (&tmp, nfa, &dfa_state->nfa_set, ctran->label); + nfa_closure_set_create (&nfa_set, nfa, &tmp, NULL); + state_set_clear (&tmp); + + state_contains = NULL; + for (state_iter = dfa->states_head; NULL != state_iter; + state_iter = state_iter->next) + { + if (0 == state_set_compare (&state_iter->nfa_set, &nfa_set)) + { + state_contains = state_iter; + break; + } + } + if (NULL == state_contains) + { + new_dfa_state = dfa_state_create (ctx, &nfa_set); + automaton_add_state (dfa, new_dfa_state); + ctran->to_state = new_dfa_state; + construct_dfa_states (ctx, nfa, dfa, new_dfa_state); + } + else + { + ctran->to_state = state_contains; + state_set_clear (&nfa_set); + } + } +} + + +/** + * Construct DFA for the given 'regex' of length 'len'. + * + * Path compression means, that for example a DFA o -> a -> b -> c -> o will be + * compressed to o -> abc -> o. Note that this parameter influences the + * non-determinism of states of the resulting NFA in the DHT (number of outgoing + * edges with the same label). For example for an application that stores IPv4 + * addresses as bitstrings it could make sense to limit the path compression to + * 4 or 8. + * + * @param regex regular expression string. + * @param len length of the regular expression. + * @param max_path_len limit the path compression length to the + * given value. If set to 1, no path compression is applied. Set to 0 for + * maximal possible path compression (generally not desireable). + * @return DFA, needs to be freed using REGEX_INTERNAL_automaton_destroy. + */ +struct REGEX_INTERNAL_Automaton * +REGEX_INTERNAL_construct_dfa (const char *regex, const size_t len, + unsigned int max_path_len) +{ + struct REGEX_INTERNAL_Context ctx; + struct REGEX_INTERNAL_Automaton *dfa; + struct REGEX_INTERNAL_Automaton *nfa; + struct REGEX_INTERNAL_StateSet nfa_start_eps_cls; + struct REGEX_INTERNAL_StateSet singleton_set; + + REGEX_INTERNAL_context_init (&ctx); + + /* Create NFA */ + nfa = REGEX_INTERNAL_construct_nfa (regex, len); + + if (NULL == nfa) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not create DFA, because NFA creation failed\n"); + return NULL; + } + + dfa = GNUNET_new (struct REGEX_INTERNAL_Automaton); + dfa->type = DFA; + dfa->regex = GNUNET_strdup (regex); + + /* Create DFA start state from epsilon closure */ + memset (&singleton_set, 0, sizeof (struct REGEX_INTERNAL_StateSet)); + state_set_append (&singleton_set, nfa->start); + nfa_closure_set_create (&nfa_start_eps_cls, nfa, &singleton_set, NULL); + state_set_clear (&singleton_set); + dfa->start = dfa_state_create (&ctx, &nfa_start_eps_cls); + automaton_add_state (dfa, dfa->start); + + construct_dfa_states (&ctx, nfa, dfa, dfa->start); + REGEX_INTERNAL_automaton_destroy (nfa); + + /* Minimize DFA */ + if (GNUNET_OK != dfa_minimize (&ctx, dfa)) + { + REGEX_INTERNAL_automaton_destroy (dfa); + return NULL; + } + + /* Create proofs and hashes for all states */ + if (GNUNET_OK != automaton_create_proofs (dfa)) + { + REGEX_INTERNAL_automaton_destroy (dfa); + return NULL; + } + + /* Compress linear DFA paths */ + if (1 != max_path_len) + dfa_compress_paths (&ctx, dfa, max_path_len); + + return dfa; +} + + +/** + * Free the memory allocated by constructing the REGEX_INTERNAL_Automaton data + * structure. + * + * @param a automaton to be destroyed + */ +void +REGEX_INTERNAL_automaton_destroy (struct REGEX_INTERNAL_Automaton *a) +{ + struct REGEX_INTERNAL_State *s; + struct REGEX_INTERNAL_State *next_state; + + if (NULL == a) + return; + + GNUNET_free_non_null (a->regex); + GNUNET_free_non_null (a->canonical_regex); + + for (s = a->states_head; NULL != s; s = next_state) + { + next_state = s->next; + GNUNET_CONTAINER_DLL_remove (a->states_head, a->states_tail, s); + automaton_destroy_state (s); + } + + GNUNET_free (a); +} + + +/** + * Evaluates the given string using the given DFA automaton + * + * @param a automaton, type must be DFA + * @param string string that should be evaluated + * + * @return 0 if string matches, non-0 otherwise + */ +static int +evaluate_dfa (struct REGEX_INTERNAL_Automaton *a, + const char *string) +{ + const char *strp; + struct REGEX_INTERNAL_State *s; + unsigned int step_len; + + if (DFA != a->type) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Tried to evaluate DFA, but NFA automaton given"); + return -1; + } + + s = a->start; + + /* If the string is empty but the starting state is accepting, we accept. */ + if ((NULL == string || 0 == strlen (string)) && s->accepting) + return 0; + + for (strp = string; NULL != strp && *strp; strp += step_len) + { + step_len = dfa_move (&s, strp); + + if (NULL == s) + break; + } + + if (NULL != s && s->accepting) + return 0; + + return 1; +} + + +/** + * Evaluates the given string using the given NFA automaton + * + * @param a automaton, type must be NFA + * @param string string that should be evaluated + * @return 0 if string matches, non-0 otherwise + */ +static int +evaluate_nfa (struct REGEX_INTERNAL_Automaton *a, + const char *string) +{ + const char *strp; + char str[2]; + struct REGEX_INTERNAL_State *s; + struct REGEX_INTERNAL_StateSet sset; + struct REGEX_INTERNAL_StateSet new_sset; + struct REGEX_INTERNAL_StateSet singleton_set; + unsigned int i; + int result; + + if (NFA != a->type) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Tried to evaluate NFA, but DFA automaton given"); + return -1; + } + + /* If the string is empty but the starting state is accepting, we accept. */ + if ((NULL == string || 0 == strlen (string)) && a->start->accepting) + return 0; + + result = 1; + memset (&singleton_set, 0, sizeof (struct REGEX_INTERNAL_StateSet)); + state_set_append (&singleton_set, a->start); + nfa_closure_set_create (&sset, a, &singleton_set, NULL); + state_set_clear (&singleton_set); + + str[1] = '\0'; + for (strp = string; NULL != strp && *strp; strp++) + { + str[0] = *strp; + nfa_closure_set_create (&new_sset, a, &sset, str); + state_set_clear (&sset); + nfa_closure_set_create (&sset, a, &new_sset, 0); + state_set_clear (&new_sset); + } + + for (i = 0; i < sset.off; i++) + { + s = sset.states[i]; + if ( (NULL != s) && (s->accepting) ) + { + result = 0; + break; + } + } + + state_set_clear (&sset); + return result; +} + + +/** + * Evaluates the given @a string against the given compiled regex @a a + * + * @param a automaton + * @param string string to check + * @return 0 if string matches, non-0 otherwise + */ +int +REGEX_INTERNAL_eval (struct REGEX_INTERNAL_Automaton *a, + const char *string) +{ + int result; + + switch (a->type) + { + case DFA: + result = evaluate_dfa (a, string); + break; + case NFA: + result = evaluate_nfa (a, string); + break; + default: + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Evaluating regex failed, automaton has no type!\n"); + result = GNUNET_SYSERR; + break; + } + + return result; +} + + +/** + * Get the canonical regex of the given automaton. + * When constructing the automaton a proof is computed for each state, + * consisting of the regular expression leading to this state. A complete + * regex for the automaton can be computed by combining these proofs. + * As of now this function is only useful for testing. + * + * @param a automaton for which the canonical regex should be returned. + * + * @return + */ +const char * +REGEX_INTERNAL_get_canonical_regex (struct REGEX_INTERNAL_Automaton *a) +{ + if (NULL == a) + return NULL; + + return a->canonical_regex; +} + + +/** + * Get the number of transitions that are contained in the given automaton. + * + * @param a automaton for which the number of transitions should be returned. + * + * @return number of transitions in the given automaton. + */ +unsigned int +REGEX_INTERNAL_get_transition_count (struct REGEX_INTERNAL_Automaton *a) +{ + unsigned int t_count; + struct REGEX_INTERNAL_State *s; + + if (NULL == a) + return 0; + + t_count = 0; + for (s = a->states_head; NULL != s; s = s->next) + t_count += s->transition_count; + + return t_count; +} + + +/** + * Get the first key for the given @a input_string. This hashes the first x bits + * of the @a input_string. + * + * @param input_string string. + * @param string_len length of the @a input_string. + * @param key pointer to where to write the hash code. + * @return number of bits of @a input_string that have been consumed + * to construct the key + */ +size_t +REGEX_INTERNAL_get_first_key (const char *input_string, + size_t string_len, + struct GNUNET_HashCode *key) +{ + size_t size; + + size = string_len < GNUNET_REGEX_INITIAL_BYTES ? string_len : + GNUNET_REGEX_INITIAL_BYTES; + if (NULL == input_string) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Given input string was NULL!\n"); + return 0; + } + GNUNET_CRYPTO_hash (input_string, size, key); + + return size; +} + + +/** + * Recursive function that calls the iterator for each synthetic start state. + * + * @param min_len minimum length of the path in the graph. + * @param max_len maximum length of the path in the graph. + * @param consumed_string string consumed by traversing the graph till this state. + * @param state current state of the automaton. + * @param iterator iterator function called for each edge. + * @param iterator_cls closure for the @a iterator function. + */ +static void +iterate_initial_edge (unsigned int min_len, + unsigned int max_len, + char *consumed_string, + struct REGEX_INTERNAL_State *state, + REGEX_INTERNAL_KeyIterator iterator, + void *iterator_cls) +{ + char *temp; + struct REGEX_INTERNAL_Transition *t; + unsigned int num_edges = state->transition_count; + struct REGEX_BLOCK_Edge edges[num_edges]; + struct REGEX_BLOCK_Edge edge[1]; + struct GNUNET_HashCode hash; + struct GNUNET_HashCode hash_new; + unsigned int cur_len; + + if (NULL != consumed_string) + cur_len = strlen (consumed_string); + else + cur_len = 0; + + if ( ( (cur_len >= min_len) || + (GNUNET_YES == state->accepting) ) && + (cur_len > 0) && + (NULL != consumed_string) ) + { + if (cur_len <= max_len) + { + if ( (NULL != state->proof) && + (0 != strcmp (consumed_string, + state->proof)) ) + { + (void) state_get_edges (state, edges); + GNUNET_CRYPTO_hash (consumed_string, + strlen (consumed_string), + &hash); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Start state for string `%s' is %s\n", + consumed_string, + GNUNET_h2s (&hash)); + iterator (iterator_cls, + &hash, + consumed_string, + state->accepting, + num_edges, edges); + } + + if ( (GNUNET_YES == state->accepting) && + (cur_len > 1) && + (state->transition_count < 1) && + (cur_len < max_len) ) + { + /* Special case for regex consisting of just a string that is shorter than + * max_len */ + edge[0].label = &consumed_string[cur_len - 1]; + edge[0].destination = state->hash; + temp = GNUNET_strdup (consumed_string); + temp[cur_len - 1] = '\0'; + GNUNET_CRYPTO_hash (temp, + cur_len - 1, + &hash_new); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Start state for short string `%s' is %s\n", + temp, + GNUNET_h2s (&hash_new)); + iterator (iterator_cls, + &hash_new, + temp, + GNUNET_NO, 1, + edge); + GNUNET_free (temp); + } + } + else /* cur_len > max_len */ + { + /* Case where the concatenated labels are longer than max_len, then split. */ + edge[0].label = &consumed_string[max_len]; + edge[0].destination = state->hash; + temp = GNUNET_strdup (consumed_string); + temp[max_len] = '\0'; + GNUNET_CRYPTO_hash (temp, max_len, &hash); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Start state at split edge `%s'-`%s` is %s\n", + temp, + edge[0].label, + GNUNET_h2s (&hash_new)); + iterator (iterator_cls, + &hash, + temp, + GNUNET_NO, + 1, + edge); + GNUNET_free (temp); + } + } + + if (cur_len < max_len) + { + for (t = state->transitions_head; NULL != t; t = t->next) + { + if (NULL != strchr (t->label, + (int) '.')) + { + /* Wildcards not allowed during starting states */ + GNUNET_break (0); + continue; + } + if (NULL != consumed_string) + GNUNET_asprintf (&temp, + "%s%s", + consumed_string, + t->label); + else + GNUNET_asprintf (&temp, + "%s", + t->label); + iterate_initial_edge (min_len, + max_len, + temp, + t->to_state, + iterator, + iterator_cls); + GNUNET_free (temp); + } + } +} + + +/** + * Iterate over all edges starting from start state of automaton 'a'. Calling + * iterator for each edge. + * + * @param a automaton. + * @param iterator iterator called for each edge. + * @param iterator_cls closure. + */ +void +REGEX_INTERNAL_iterate_all_edges (struct REGEX_INTERNAL_Automaton *a, + REGEX_INTERNAL_KeyIterator iterator, + void *iterator_cls) +{ + struct REGEX_INTERNAL_State *s; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Iterating over starting edges\n"); + iterate_initial_edge (GNUNET_REGEX_INITIAL_BYTES, + GNUNET_REGEX_INITIAL_BYTES, + NULL, a->start, + iterator, iterator_cls); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Iterating over DFA edges\n"); + for (s = a->states_head; NULL != s; s = s->next) + { + struct REGEX_BLOCK_Edge edges[s->transition_count]; + unsigned int num_edges; + + num_edges = state_get_edges (s, edges); + if ( ( (NULL != s->proof) && + (0 < strlen (s->proof)) ) || s->accepting) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Creating DFA edges at `%s' under key %s\n", + s->proof, + GNUNET_h2s (&s->hash)); + iterator (iterator_cls, &s->hash, s->proof, + s->accepting, + num_edges, edges); + } + s->marked = GNUNET_NO; + } +} + + +/** + * Struct to hold all the relevant state information in the HashMap. + * + * Contains the same info as the Regex Iterator parametes except the key, + * which comes directly from the HashMap iterator. + */ +struct temporal_state_store { + int reachable; + char *proof; + int accepting; + int num_edges; + struct REGEX_BLOCK_Edge *edges; +}; + + +/** + * Store regex iterator and cls in one place to pass to the hashmap iterator. + */ +struct client_iterator { + REGEX_INTERNAL_KeyIterator iterator; + void *iterator_cls; +}; + + +/** + * Iterator over all edges of a dfa. Stores all of them in a HashMap + * for later reachability marking. + * + * @param cls Closure (HashMap) + * @param key hash for current state. + * @param proof proof for current state + * @param accepting GNUNET_YES if this is an accepting state, GNUNET_NO if not. + * @param num_edges number of edges leaving current state. + * @param edges edges leaving current state. + */ +static void +store_all_states (void *cls, + const struct GNUNET_HashCode *key, + const char *proof, + int accepting, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges) +{ + struct GNUNET_CONTAINER_MultiHashMap *hm = cls; + struct temporal_state_store *tmp; + size_t edges_size; + + tmp = GNUNET_new (struct temporal_state_store); + tmp->reachable = GNUNET_NO; + tmp->proof = GNUNET_strdup (proof); + tmp->accepting = accepting; + tmp->num_edges = num_edges; + edges_size = sizeof (struct REGEX_BLOCK_Edge) * num_edges; + tmp->edges = GNUNET_malloc (edges_size); + GNUNET_memcpy(tmp->edges, edges, edges_size); + GNUNET_CONTAINER_multihashmap_put (hm, key, tmp, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST); +} + + +/** + * Mark state as reachable and call recursively on all its edges. + * + * If already marked as reachable, do nothing. + * + * @param state State to mark as reachable. + * @param hm HashMap which stores all the states indexed by key. + */ +static void +mark_as_reachable (struct temporal_state_store *state, + struct GNUNET_CONTAINER_MultiHashMap *hm) +{ + struct temporal_state_store *child; + unsigned int i; + + if (GNUNET_YES == state->reachable) + /* visited */ + return; + + state->reachable = GNUNET_YES; + for (i = 0; i < state->num_edges; i++) + { + child = GNUNET_CONTAINER_multihashmap_get (hm, + &state->edges[i].destination); + if (NULL == child) + { + GNUNET_break (0); + continue; + } + mark_as_reachable (child, hm); + } +} + + +/** + * Iterator over hash map entries to mark the ones that are reachable. + * + * @param cls closure + * @param key current key code + * @param value value in the hash map + * @return #GNUNET_YES if we should continue to iterate, + * #GNUNET_NO if not. + */ +static int +reachability_iterator (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct GNUNET_CONTAINER_MultiHashMap *hm = cls; + struct temporal_state_store *state = value; + + if (GNUNET_YES == state->reachable) + /* already visited and marked */ + return GNUNET_YES; + + if (GNUNET_REGEX_INITIAL_BYTES > strlen (state->proof) && + GNUNET_NO == state->accepting) + /* not directly reachable */ + return GNUNET_YES; + + mark_as_reachable (state, hm); + return GNUNET_YES; +} + + +/** + * Iterator over hash map entries. + * Calling the callback on the ones marked as reachables. + * + * @param cls closure + * @param key current key code + * @param value value in the hash map + * @return #GNUNET_YES if we should continue to iterate, + * #GNUNET_NO if not. + */ +static int +iterate_reachables (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct client_iterator *ci = cls; + struct temporal_state_store *state = value; + + if (GNUNET_YES == state->reachable) + { + ci->iterator (ci->iterator_cls, key, + state->proof, state->accepting, + state->num_edges, state->edges); + } + GNUNET_free (state->edges); + GNUNET_free (state->proof); + GNUNET_free (state); + return GNUNET_YES; + +} + +/** + * Iterate over all edges of automaton 'a' that are reachable from a state with + * a proof of at least GNUNET_REGEX_INITIAL_BYTES characters. + * + * Call the iterator for each such edge. + * + * @param a automaton. + * @param iterator iterator called for each reachable edge. + * @param iterator_cls closure. + */ +void +REGEX_INTERNAL_iterate_reachable_edges (struct REGEX_INTERNAL_Automaton *a, + REGEX_INTERNAL_KeyIterator iterator, + void *iterator_cls) +{ + struct GNUNET_CONTAINER_MultiHashMap *hm; + struct client_iterator ci; + + hm = GNUNET_CONTAINER_multihashmap_create (a->state_count * 2, GNUNET_NO); + ci.iterator = iterator; + ci.iterator_cls = iterator_cls; + + REGEX_INTERNAL_iterate_all_edges (a, &store_all_states, hm); + GNUNET_CONTAINER_multihashmap_iterate (hm, &reachability_iterator, hm); + GNUNET_CONTAINER_multihashmap_iterate (hm, &iterate_reachables, &ci); + + GNUNET_CONTAINER_multihashmap_destroy (hm); +} + + +/* end of regex_internal.c */ diff --git a/src/regex/regex_internal.h b/src/regex/regex_internal.h new file mode 100644 index 000000000..d52479ffe --- /dev/null +++ b/src/regex/regex_internal.h @@ -0,0 +1,453 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file src/regex/regex_internal.h + * @brief common internal definitions for regex library. + * @author Maximilian Szengel + */ +#ifndef REGEX_INTERNAL_H +#define REGEX_INTERNAL_H + +#include "regex_internal_lib.h" + +#ifdef __cplusplus +extern "C" +{ +#if 0 /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + +/** + * char array of literals that are allowed inside a regex (apart from the + * operators) + */ +#define ALLOWED_LITERALS "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + +/** + * Transition between two states. Transitions are stored at the states from + * which they origin ('from_state'). Each state can have 0-n transitions. + * If label is NULL, this is considered to be an epsilon transition. + */ +struct REGEX_INTERNAL_Transition +{ + /** + * This is a linked list. + */ + struct REGEX_INTERNAL_Transition *prev; + + /** + * This is a linked list. + */ + struct REGEX_INTERNAL_Transition *next; + + /** + * Unique id of this transition. + */ + unsigned int id; + + /** + * Label for this transition. This is basically the edge label for the graph. + */ + char *label; + + /** + * State to which this transition leads. + */ + struct REGEX_INTERNAL_State *to_state; + + /** + * State from which this transition origins. + */ + struct REGEX_INTERNAL_State *from_state; +}; + + +/** + * A state. Can be used in DFA and NFA automatons. + */ +struct REGEX_INTERNAL_State; + + +/** + * Set of states. + */ +struct REGEX_INTERNAL_StateSet +{ + /** + * Array of states. + */ + struct REGEX_INTERNAL_State **states; + + /** + * Number of entries in *use* in the 'states' array. + */ + unsigned int off; + + /** + * Length of the 'states' array. + */ + unsigned int size; +}; + + +/** + * A state. Can be used in DFA and NFA automatons. + */ +struct REGEX_INTERNAL_State +{ + /** + * This is a linked list to keep states in an automaton. + */ + struct REGEX_INTERNAL_State *prev; + + /** + * This is a linked list to keep states in an automaton. + */ + struct REGEX_INTERNAL_State *next; + + /** + * This is a multi DLL for StateSet_MDLL. + */ + struct REGEX_INTERNAL_State *prev_SS; + + /** + * This is a multi DLL for StateSet_MDLL. + */ + struct REGEX_INTERNAL_State *next_SS; + + /** + * This is a multi DLL for StateSet_MDLL Stack. + */ + struct REGEX_INTERNAL_State *prev_ST; + + /** + * This is a multi DLL for StateSet_MDLL Stack. + */ + struct REGEX_INTERNAL_State *next_ST; + + /** + * Unique state id. + */ + unsigned int id; + + /** + * Unique state id that is used for traversing the automaton. It is guaranteed + * to be > 0 and < state_count. + */ + unsigned int traversal_id; + + /** + * If this is an accepting state or not. + */ + int accepting; + + /** + * Marking of the state. This is used for marking all visited states when + * traversing all states of an automaton and for cases where the state id + * cannot be used (dfa minimization). + */ + int marked; + + /** + * Marking the state as contained. This is used for checking, if the state is + * contained in a set in constant time. + */ + int contained; + + /** + * Marking the state as part of an SCC (Strongly Connected Component). All + * states with the same scc_id are part of the same SCC. scc_id is 0, if state + * is not a part of any SCC. + */ + unsigned int scc_id; + + /** + * Used for SCC detection. + */ + int index; + + /** + * Used for SCC detection. + */ + int lowlink; + + /** + * Human readable name of the state. Used for debugging and graph + * creation. + */ + char *name; + + /** + * Hash of the state. + */ + struct GNUNET_HashCode hash; + + /** + * Linear state ID accquired by depth-first-search. This ID should be used for + * storing information about the state in an array, because the 'id' of the + * state is not guaranteed to be linear. The 'dfs_id' is guaranteed to be > 0 + * and < 'state_count'. + */ + unsigned int dfs_id; + + /** + * Proof for this state. + */ + char *proof; + + /** + * Number of transitions from this state to other states. + */ + unsigned int transition_count; + + /** + * DLL of transitions. + */ + struct REGEX_INTERNAL_Transition *transitions_head; + + /** + * DLL of transitions. + */ + struct REGEX_INTERNAL_Transition *transitions_tail; + + /** + * Number of incoming transitions. Used for compressing DFA paths. + */ + unsigned int incoming_transition_count; + + /** + * Set of states on which this state is based on. Used when creating a DFA out + * of several NFA states. + */ + struct REGEX_INTERNAL_StateSet nfa_set; +}; + + +/** + * Type of an automaton. + */ +enum REGEX_INTERNAL_AutomatonType +{ + NFA, + DFA +}; + + +/** + * Automaton representation. + */ +struct REGEX_INTERNAL_Automaton +{ + /** + * Linked list of NFAs used for partial NFA creation. + */ + struct REGEX_INTERNAL_Automaton *prev; + + /** + * Linked list of NFAs used for partial NFA creation. + */ + struct REGEX_INTERNAL_Automaton *next; + + /** + * First state of the automaton. This is mainly used for constructing an NFA, + * where each NFA itself consists of one or more NFAs linked together. + */ + struct REGEX_INTERNAL_State *start; + + /** + * End state of the partial NFA. This is undefined for DFAs + */ + struct REGEX_INTERNAL_State *end; + + /** + * Number of states in the automaton. + */ + unsigned int state_count; + + /** + * DLL of states. + */ + struct REGEX_INTERNAL_State *states_head; + + /** + * DLL of states + */ + struct REGEX_INTERNAL_State *states_tail; + + /** + * Type of the automaton. + */ + enum REGEX_INTERNAL_AutomatonType type; + + /** + * Regex + */ + char *regex; + + /** + * Canonical regex (result of RX->NFA->DFA->RX) + */ + char *canonical_regex; + + /** + * GNUNET_YES, if multi strides have been added to the Automaton. + */ + int is_multistrided; +}; + + +/** + * Construct an NFA by parsing the regex string of length 'len'. + * + * @param regex regular expression string. + * @param len length of the string. + * + * @return NFA, needs to be freed using REGEX_INTERNAL_automaton_destroy. + */ +struct REGEX_INTERNAL_Automaton * +REGEX_INTERNAL_construct_nfa (const char *regex, const size_t len); + + +/** + * Function that get's passed to automaton traversal and is called before each + * next traversal from state 's' using transition 't' to check if traversal + * should proceed. Return GNUNET_NO to stop traversal or GNUNET_YES to continue. + * + * @param cls closure for the check. + * @param s current state in the traversal. + * @param t current transition from state 's' that will be used for the next + * step. + * + * @return GNUNET_YES to proceed traversal, GNUNET_NO to stop. + */ +typedef int (*REGEX_INTERNAL_traverse_check) (void *cls, + struct REGEX_INTERNAL_State * s, + struct REGEX_INTERNAL_Transition * t); + + +/** + * Function that is called with each state, when traversing an automaton. + * + * @param cls closure. + * @param count current count of the state, from 0 to a->state_count -1. + * @param s state. + */ +typedef void (*REGEX_INTERNAL_traverse_action) (void *cls, + const unsigned int count, + struct REGEX_INTERNAL_State * s); + + +/** + * Traverses the given automaton using depth-first-search (DFS) from it's start + * state, visiting all reachable states and calling 'action' on each one of + * them. + * + * @param a automaton to be traversed. + * @param start start state, pass a->start or NULL to traverse the whole automaton. + * @param check function that is checked before advancing on each transition + * in the DFS. + * @param check_cls closure for check. + * @param action action to be performed on each state. + * @param action_cls closure for action + */ +void +REGEX_INTERNAL_automaton_traverse (const struct REGEX_INTERNAL_Automaton *a, + struct REGEX_INTERNAL_State *start, + REGEX_INTERNAL_traverse_check check, + void *check_cls, + REGEX_INTERNAL_traverse_action action, + void *action_cls); + +/** + * Get the canonical regex of the given automaton. + * When constructing the automaton a proof is computed for each state, + * consisting of the regular expression leading to this state. A complete + * regex for the automaton can be computed by combining these proofs. + * As of now this function is only useful for testing. + * + * @param a automaton for which the canonical regex should be returned. + * + * @return canonical regex string. + */ +const char * +REGEX_INTERNAL_get_canonical_regex (struct REGEX_INTERNAL_Automaton *a); + + +/** + * Get the number of transitions that are contained in the given automaton. + * + * @param a automaton for which the number of transitions should be returned. + * + * @return number of transitions in the given automaton. + */ +unsigned int +REGEX_INTERNAL_get_transition_count (struct REGEX_INTERNAL_Automaton *a); + + +/** + * Context that contains an id counter for states and transitions as well as a + * DLL of automatons used as a stack for NFA construction. + */ +struct REGEX_INTERNAL_Context +{ + /** + * Unique state id. + */ + unsigned int state_id; + + /** + * Unique transition id. + */ + unsigned int transition_id; + + /** + * DLL of REGEX_INTERNAL_Automaton's used as a stack. + */ + struct REGEX_INTERNAL_Automaton *stack_head; + + /** + * DLL of REGEX_INTERNAL_Automaton's used as a stack. + */ + struct REGEX_INTERNAL_Automaton *stack_tail; +}; + + +/** + * Adds multi-strided transitions to the given 'dfa'. + * + * @param regex_ctx regex context needed to add transitions to the automaton. + * @param dfa DFA to which the multi strided transitions should be added. + * @param stride_len length of the strides. + */ +void +REGEX_INTERNAL_dfa_add_multi_strides (struct REGEX_INTERNAL_Context *regex_ctx, + struct REGEX_INTERNAL_Automaton *dfa, + const unsigned int stride_len); + + + +#if 0 /* keep Emacsens' auto-indent happy */ +{ +#endif +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/regex/regex_internal_dht.c b/src/regex/regex_internal_dht.c new file mode 100644 index 000000000..2555ef1df --- /dev/null +++ b/src/regex/regex_internal_dht.c @@ -0,0 +1,829 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2015 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file src/regex/regex_internal_dht.c + * @brief library to announce regexes in the network and match strings + * against published regexes. + * @author Bartlomiej Polot + */ +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_block_lib.h" +#include "gnunet_dht_service.h" +#include "gnunet_statistics_service.h" +#include "gnunet_constants.h" +#include "gnunet_signatures.h" + + +#define LOG(kind,...) GNUNET_log_from (kind,"regex-dht",__VA_ARGS__) + +/** + * DHT replication level to use. + */ +#define DHT_REPLICATION 5 + +/** + * DHT record lifetime to use. + */ +#define DHT_TTL GNUNET_TIME_UNIT_HOURS + +/** + * DHT options to set. + */ +#define DHT_OPT GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE + + +/** + * Handle to store cached data about a regex announce. + */ +struct REGEX_INTERNAL_Announcement +{ + /** + * DHT handle to use, must be initialized externally. + */ + struct GNUNET_DHT_Handle *dht; + + /** + * Regular expression. + */ + const char *regex; + + /** + * Automaton representation of the regex (expensive to build). + */ + struct REGEX_INTERNAL_Automaton *dfa; + + /** + * Our private key. + */ + const struct GNUNET_CRYPTO_EddsaPrivateKey *priv; + + /** + * Optional statistics handle to report usage. Can be NULL. + */ + struct GNUNET_STATISTICS_Handle *stats; +}; + + +/** + * Regex callback iterator to store own service description in the DHT. + * + * @param cls closure. + * @param key hash for current state. + * @param proof proof for current state. + * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not. + * @param num_edges number of edges leaving current state. + * @param edges edges leaving current state. + */ +static void +regex_iterator (void *cls, + const struct GNUNET_HashCode *key, + const char *proof, + int accepting, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges) +{ + struct REGEX_INTERNAL_Announcement *h = cls; + struct RegexBlock *block; + size_t size; + unsigned int i; + + LOG (GNUNET_ERROR_TYPE_INFO, + "DHT PUT for state %s with proof `%s' and %u edges:\n", + GNUNET_h2s (key), + proof, + num_edges); + for (i = 0; i < num_edges; i++) + { + LOG (GNUNET_ERROR_TYPE_INFO, + "Edge %u `%s' towards %s\n", + i, + edges[i].label, + GNUNET_h2s (&edges[i].destination)); + } + if (GNUNET_YES == accepting) + { + struct RegexAcceptBlock ab; + + LOG (GNUNET_ERROR_TYPE_INFO, + "State %s is accepting, putting own id\n", + GNUNET_h2s (key)); + size = sizeof (struct RegexAcceptBlock); + ab.purpose.size = ntohl (sizeof (struct GNUNET_CRYPTO_EccSignaturePurpose) + + sizeof (struct GNUNET_TIME_AbsoluteNBO) + + sizeof (struct GNUNET_HashCode)); + ab.purpose.purpose = ntohl (GNUNET_SIGNATURE_PURPOSE_REGEX_ACCEPT); + ab.expiration_time = GNUNET_TIME_absolute_hton (GNUNET_TIME_relative_to_absolute (GNUNET_CONSTANTS_DHT_MAX_EXPIRATION)); + ab.key = *key; + GNUNET_CRYPTO_eddsa_key_get_public (h->priv, + &ab.peer.public_key); + GNUNET_assert (GNUNET_OK == + GNUNET_CRYPTO_eddsa_sign (h->priv, + &ab.purpose, + &ab.signature)); + + GNUNET_STATISTICS_update (h->stats, "# regex accepting blocks stored", + 1, GNUNET_NO); + GNUNET_STATISTICS_update (h->stats, "# regex accepting block bytes stored", + sizeof (struct RegexAcceptBlock), GNUNET_NO); + (void) + GNUNET_DHT_put (h->dht, key, + DHT_REPLICATION, + DHT_OPT | GNUNET_DHT_RO_RECORD_ROUTE, + GNUNET_BLOCK_TYPE_REGEX_ACCEPT, + size, + &ab, + GNUNET_TIME_relative_to_absolute (DHT_TTL), + NULL, NULL); + } + block = REGEX_BLOCK_create (proof, + num_edges, + edges, + accepting, + &size); + if (NULL == block) + return; + (void) GNUNET_DHT_put (h->dht, + key, + DHT_REPLICATION, + DHT_OPT, + GNUNET_BLOCK_TYPE_REGEX, + size, + block, + GNUNET_TIME_relative_to_absolute (DHT_TTL), + NULL, + NULL); + GNUNET_STATISTICS_update (h->stats, + "# regex blocks stored", + 1, + GNUNET_NO); + GNUNET_STATISTICS_update (h->stats, + "# regex block bytes stored", + size, + GNUNET_NO); + GNUNET_free (block); +} + + +/** + * Announce a regular expression: put all states of the automaton in the DHT. + * Does not free resources, must call #REGEX_INTERNAL_announce_cancel() for that. + * + * @param dht An existing and valid DHT service handle. CANNOT be NULL. + * @param priv our private key, must remain valid until the announcement is cancelled + * @param regex Regular expression to announce. + * @param compression How many characters per edge can we squeeze? + * @param stats Optional statistics handle to report usage. Can be NULL. + * @return Handle to reuse o free cached resources. + * Must be freed by calling #REGEX_INTERNAL_announce_cancel(). + */ +struct REGEX_INTERNAL_Announcement * +REGEX_INTERNAL_announce (struct GNUNET_DHT_Handle *dht, + const struct GNUNET_CRYPTO_EddsaPrivateKey *priv, + const char *regex, + uint16_t compression, + struct GNUNET_STATISTICS_Handle *stats) +{ + struct REGEX_INTERNAL_Announcement *h; + + GNUNET_assert (NULL != dht); + h = GNUNET_new (struct REGEX_INTERNAL_Announcement); + h->regex = regex; + h->dht = dht; + h->stats = stats; + h->priv = priv; + h->dfa = REGEX_INTERNAL_construct_dfa (regex, strlen (regex), compression); + REGEX_INTERNAL_reannounce (h); + return h; +} + + +/** + * Announce again a regular expression previously announced. + * Does use caching to speed up process. + * + * @param h Handle returned by a previous #REGEX_INTERNAL_announce call(). + */ +void +REGEX_INTERNAL_reannounce (struct REGEX_INTERNAL_Announcement *h) +{ + GNUNET_assert (NULL != h->dfa); /* make sure to call announce first */ + LOG (GNUNET_ERROR_TYPE_INFO, + "REGEX_INTERNAL_reannounce: %s\n", + h->regex); + REGEX_INTERNAL_iterate_reachable_edges (h->dfa, + ®ex_iterator, + h); +} + + +/** + * Clear all cached data used by a regex announce. + * Does not close DHT connection. + * + * @param h Handle returned by a previous #REGEX_INTERNAL_announce() call. + */ +void +REGEX_INTERNAL_announce_cancel (struct REGEX_INTERNAL_Announcement *h) +{ + REGEX_INTERNAL_automaton_destroy (h->dfa); + GNUNET_free (h); +} + + +/******************************************************************************/ + + +/** + * Struct to keep state of running searches that have consumed a part of + * the inital string. + */ +struct RegexSearchContext +{ + /** + * Part of the description already consumed by + * this particular search branch. + */ + size_t position; + + /** + * Information about the search. + */ + struct REGEX_INTERNAL_Search *info; + + /** + * We just want to look for one edge, the longer the better. + * Keep its length. + */ + unsigned int longest_match; + + /** + * Destination hash of the longest match. + */ + struct GNUNET_HashCode hash; +}; + + +/** + * Type of values in `dht_get_results`. + */ +struct Result +{ + /** + * Number of bytes in data. + */ + size_t size; + + /** + * The raw result data. + */ + const void *data; +}; + + +/** + * Struct to keep information of searches of services described by a regex + * using a user-provided string service description. + */ +struct REGEX_INTERNAL_Search +{ + /** + * DHT handle to use, must be initialized externally. + */ + struct GNUNET_DHT_Handle *dht; + + /** + * Optional statistics handle to report usage. Can be NULL. + */ + struct GNUNET_STATISTICS_Handle *stats; + + /** + * User provided description of the searched service. + */ + char *description; + + /** + * Running DHT GETs. + */ + struct GNUNET_CONTAINER_MultiHashMap *dht_get_handles; + + /** + * Results from running DHT GETs, values are of type + * 'struct Result'. + */ + struct GNUNET_CONTAINER_MultiHashMap *dht_get_results; + + /** + * Contexts, for each running DHT GET. Free all on end of search. + */ + struct RegexSearchContext **contexts; + + /** + * Number of contexts (branches/steps in search). + */ + unsigned int n_contexts; + + /** + * @param callback Callback for found peers. + */ + REGEX_INTERNAL_Found callback; + + /** + * @param callback_cls Closure for @c callback. + */ + void *callback_cls; +}; + + +/** + * Jump to the next edge, with the longest matching token. + * + * @param block Block found in the DHT. + * @param size Size of the block. + * @param ctx Context of the search. + */ +static void +regex_next_edge (const struct RegexBlock *block, + size_t size, + struct RegexSearchContext *ctx); + + +/** + * Function to process DHT string to regex matching. + * Called on each result obtained for the DHT search. + * + * @param cls Closure (search context). + * @param exp When will this value expire. + * @param key Key of the result. + * @param get_path Path of the get request. + * @param get_path_length Lenght of get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the put_path. + * @param type Type of the result. + * @param size Number of bytes in data. + * @param data Pointer to the result data. + */ +static void +dht_get_string_accept_handler (void *cls, struct GNUNET_TIME_Absolute exp, + const struct GNUNET_HashCode *key, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length, + enum GNUNET_BLOCK_Type type, + size_t size, const void *data) +{ + const struct RegexAcceptBlock *block = data; + struct RegexSearchContext *ctx = cls; + struct REGEX_INTERNAL_Search *info = ctx->info; + + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Regex result accept for %s (key %s)\n", + info->description, GNUNET_h2s(key)); + + GNUNET_STATISTICS_update (info->stats, + "# regex accepting blocks found", + 1, GNUNET_NO); + GNUNET_STATISTICS_update (info->stats, + "# regex accepting block bytes found", + size, GNUNET_NO); + info->callback (info->callback_cls, + &block->peer, + get_path, get_path_length, + put_path, put_path_length); +} + + +/** + * Find a path to a peer that offers a regex service compatible + * with a given string. + * + * @param key The key of the accepting state. + * @param ctx Context containing info about the string, tunnel, etc. + */ +static void +regex_find_path (const struct GNUNET_HashCode *key, + struct RegexSearchContext *ctx) +{ + struct GNUNET_DHT_GetHandle *get_h; + + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Accept state found, now searching for paths to %s\n", + GNUNET_h2s (key), + (unsigned int) ctx->position); + get_h = GNUNET_DHT_get_start (ctx->info->dht, /* handle */ + GNUNET_BLOCK_TYPE_REGEX_ACCEPT, /* type */ + key, /* key to search */ + DHT_REPLICATION, /* replication level */ + DHT_OPT | GNUNET_DHT_RO_RECORD_ROUTE, + NULL, /* xquery */ // FIXME BLOOMFILTER + 0, /* xquery bits */ // FIXME BLOOMFILTER SIZE + &dht_get_string_accept_handler, ctx); + GNUNET_break (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put(ctx->info->dht_get_handles, + key, + get_h, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); +} + + +/** + * Function to process DHT string to regex matching. + * Called on each result obtained for the DHT search. + * + * @param cls closure (search context) + * @param exp when will this value expire + * @param key key of the result + * @param get_path path of the get request (not used) + * @param get_path_length length of @a get_path (not used) + * @param put_path path of the put request (not used) + * @param put_path_length length of the @a put_path (not used) + * @param type type of the result + * @param size number of bytes in data + * @param data pointer to the result data + * + * TODO: re-issue the request after certain time? cancel after X results? + */ +static void +dht_get_string_handler (void *cls, struct GNUNET_TIME_Absolute exp, + const struct GNUNET_HashCode *key, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length, + enum GNUNET_BLOCK_Type type, + size_t size, const void *data) +{ + const struct RegexBlock *block = data; + struct RegexSearchContext *ctx = cls; + struct REGEX_INTERNAL_Search *info = ctx->info; + size_t len; + struct Result *copy; + + LOG (GNUNET_ERROR_TYPE_INFO, + "DHT GET result for %s (%s)\n", + GNUNET_h2s (key), ctx->info->description); + copy = GNUNET_malloc (sizeof (struct Result) + size); + copy->size = size; + copy->data = ©[1]; + GNUNET_memcpy (©[1], block, size); + GNUNET_break (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (info->dht_get_results, + key, copy, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); + len = strlen (info->description); + if (len == ctx->position) // String processed + { + if (GNUNET_YES == GNUNET_BLOCK_is_accepting (block, size)) + { + regex_find_path (key, ctx); + } + else + { + LOG (GNUNET_ERROR_TYPE_INFO, "block not accepting!\n"); + /* FIXME REGEX this block not successful, wait for more? start timeout? */ + } + return; + } + regex_next_edge (block, size, ctx); +} + + +/** + * Iterator over found existing cadet regex blocks that match an ongoing search. + * + * @param cls Closure (current context)- + * @param key Current key code (key for cached block). + * @param value Value in the hash map (cached RegexBlock). + * @return #GNUNET_YES: we should always continue to iterate. + */ +static int +regex_result_iterator (void *cls, + const struct GNUNET_HashCode * key, + void *value) +{ + struct Result *result = value; + const struct RegexBlock *block = result->data; + struct RegexSearchContext *ctx = cls; + + if ( (GNUNET_YES == + GNUNET_BLOCK_is_accepting (block, result->size)) && + (ctx->position == strlen (ctx->info->description)) ) + { + LOG (GNUNET_ERROR_TYPE_INFO, + "Found accepting known block\n"); + regex_find_path (key, ctx); + return GNUNET_YES; // We found an accept state! + } + LOG (GNUNET_ERROR_TYPE_DEBUG, + "* %u, %u, [%u]\n", + ctx->position, + strlen (ctx->info->description), + GNUNET_BLOCK_is_accepting (block, result->size)); + regex_next_edge (block, result->size, ctx); + + GNUNET_STATISTICS_update (ctx->info->stats, "# regex cadet blocks iterated", + 1, GNUNET_NO); + + return GNUNET_YES; +} + + +/** + * Iterator over edges in a regex block retrieved from the DHT. + * + * @param cls Closure (context of the search). + * @param token Token that follows to next state. + * @param len Lenght of token. + * @param key Hash of next state. + * @return #GNUNET_YES if should keep iterating, #GNUNET_NO otherwise. + */ +static int +regex_edge_iterator (void *cls, + const char *token, + size_t len, + const struct GNUNET_HashCode *key) +{ + struct RegexSearchContext *ctx = cls; + struct REGEX_INTERNAL_Search *info = ctx->info; + const char *current; + size_t current_len; + + GNUNET_STATISTICS_update (info->stats, "# regex edges iterated", + 1, GNUNET_NO); + current = &info->description[ctx->position]; + current_len = strlen (info->description) - ctx->position; + if (len > current_len) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, "Token too long, END\n"); + return GNUNET_YES; + } + if (0 != strncmp (current, token, len)) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, "Token doesn't match, END\n"); + return GNUNET_YES; + } + + if (len > ctx->longest_match) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, "Token is longer, KEEP\n"); + ctx->longest_match = len; + ctx->hash = *key; + } + else + { + LOG (GNUNET_ERROR_TYPE_DEBUG, "Token is not longer, IGNORE\n"); + } + + LOG (GNUNET_ERROR_TYPE_DEBUG, "* End of regex edge iterator\n"); + return GNUNET_YES; +} + + +/** + * Jump to the next edge, with the longest matching token. + * + * @param block Block found in the DHT. + * @param size Size of the block. + * @param ctx Context of the search. + */ +static void +regex_next_edge (const struct RegexBlock *block, + size_t size, + struct RegexSearchContext *ctx) +{ + struct RegexSearchContext *new_ctx; + struct REGEX_INTERNAL_Search *info = ctx->info; + struct GNUNET_DHT_GetHandle *get_h; + struct GNUNET_HashCode *hash; + const char *rest; + int result; + + LOG (GNUNET_ERROR_TYPE_DEBUG, "Next edge\n"); + /* Find the longest match for the current string position, + * among tokens in the given block */ + ctx->longest_match = 0; + result = REGEX_BLOCK_iterate (block, size, + ®ex_edge_iterator, ctx); + GNUNET_break (GNUNET_OK == result); + + /* Did anything match? */ + if (0 == ctx->longest_match) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, + "no match in block\n"); + return; + } + + hash = &ctx->hash; + new_ctx = GNUNET_new (struct RegexSearchContext); + new_ctx->info = info; + new_ctx->position = ctx->position + ctx->longest_match; + GNUNET_array_append (info->contexts, info->n_contexts, new_ctx); + + /* Check whether we already have a DHT GET running for it */ + if (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_contains (info->dht_get_handles, hash)) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, + "GET for %s running, END\n", + GNUNET_h2s (hash)); + GNUNET_CONTAINER_multihashmap_get_multiple (info->dht_get_results, + hash, + ®ex_result_iterator, + new_ctx); + return; /* We are already looking for it */ + } + + GNUNET_STATISTICS_update (info->stats, "# regex nodes traversed", + 1, GNUNET_NO); + + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Following edges at %s for offset %u in `%s'\n", + GNUNET_h2s (hash), + (unsigned int) ctx->position, + info->description); + rest = &new_ctx->info->description[new_ctx->position]; + get_h = + GNUNET_DHT_get_start (info->dht, /* handle */ + GNUNET_BLOCK_TYPE_REGEX, /* type */ + hash, /* key to search */ + DHT_REPLICATION, /* replication level */ + DHT_OPT, + rest, /* xquery */ + strlen (rest) + 1, /* xquery bits */ + &dht_get_string_handler, new_ctx); + if (GNUNET_OK != + GNUNET_CONTAINER_multihashmap_put(info->dht_get_handles, + hash, + get_h, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST)) + { + GNUNET_break (0); + return; + } +} + + +/** + * Search for a peer offering a regex matching certain string in the DHT. + * The search runs until #REGEX_INTERNAL_search_cancel() is called, even if results + * are returned. + * + * @param dht An existing and valid DHT service handle. + * @param string String to match against the regexes in the DHT. + * @param callback Callback for found peers. + * @param callback_cls Closure for @c callback. + * @param stats Optional statistics handle to report usage. Can be NULL. + * @return Handle to stop search and free resources. + * Must be freed by calling #REGEX_INTERNAL_search_cancel(). + */ +struct REGEX_INTERNAL_Search * +REGEX_INTERNAL_search (struct GNUNET_DHT_Handle *dht, + const char *string, + REGEX_INTERNAL_Found callback, + void *callback_cls, + struct GNUNET_STATISTICS_Handle *stats) +{ + struct REGEX_INTERNAL_Search *h; + struct GNUNET_DHT_GetHandle *get_h; + struct RegexSearchContext *ctx; + struct GNUNET_HashCode key; + size_t size; + size_t len; + + /* Initialize handle */ + GNUNET_assert (NULL != dht); + GNUNET_assert (NULL != callback); + h = GNUNET_new (struct REGEX_INTERNAL_Search); + h->dht = dht; + h->description = GNUNET_strdup (string); + h->callback = callback; + h->callback_cls = callback_cls; + h->stats = stats; + h->dht_get_handles = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO); + h->dht_get_results = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO); + + /* Initialize context */ + len = strlen (string); + size = REGEX_INTERNAL_get_first_key (string, len, &key); + LOG (GNUNET_ERROR_TYPE_INFO, + "Initial key for `%s' is %s (based on `%.*s')\n", + string, + GNUNET_h2s (&key), + size, + string); + ctx = GNUNET_new (struct RegexSearchContext); + ctx->position = size; + ctx->info = h; + GNUNET_array_append (h->contexts, + h->n_contexts, + ctx); + /* Start search in DHT */ + get_h = GNUNET_DHT_get_start (h->dht, /* handle */ + GNUNET_BLOCK_TYPE_REGEX, /* type */ + &key, /* key to search */ + DHT_REPLICATION, /* replication level */ + DHT_OPT, + &h->description[size], /* xquery */ + // FIXME add BLOOMFILTER to exclude filtered peers + len + 1 - size, /* xquery bits */ + // FIXME add BLOOMFILTER SIZE + &dht_get_string_handler, ctx); + GNUNET_break ( + GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (h->dht_get_handles, + &key, + get_h, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST) + ); + + return h; +} + + +/** + * Iterator over hash map entries to cancel DHT GET requests after a + * successful connect_by_string. + * + * @param cls Closure (unused). + * @param key Current key code (unused). + * @param value Value in the hash map (get handle). + * @return #GNUNET_YES if we should continue to iterate, + * #GNUNET_NO if not. + */ +static int +regex_cancel_dht_get (void *cls, + const struct GNUNET_HashCode * key, + void *value) +{ + struct GNUNET_DHT_GetHandle *h = value; + + GNUNET_DHT_get_stop (h); + return GNUNET_YES; +} + + +/** + * Iterator over hash map entries to free CadetRegexBlocks stored during the + * search for connect_by_string. + * + * @param cls Closure (unused). + * @param key Current key code (unused). + * @param value CadetRegexBlock in the hash map. + * @return #GNUNET_YES if we should continue to iterate, + * #GNUNET_NO if not. + */ +static int +regex_free_result (void *cls, + const struct GNUNET_HashCode * key, + void *value) +{ + GNUNET_free (value); + return GNUNET_YES; +} + + +/** + * Cancel an ongoing regex search in the DHT and free all resources. + * + * @param h the search context. + */ +void +REGEX_INTERNAL_search_cancel (struct REGEX_INTERNAL_Search *h) +{ + unsigned int i; + + GNUNET_free (h->description); + GNUNET_CONTAINER_multihashmap_iterate (h->dht_get_handles, + ®ex_cancel_dht_get, NULL); + GNUNET_CONTAINER_multihashmap_iterate (h->dht_get_results, + ®ex_free_result, NULL); + GNUNET_CONTAINER_multihashmap_destroy (h->dht_get_results); + GNUNET_CONTAINER_multihashmap_destroy (h->dht_get_handles); + if (0 < h->n_contexts) + { + for (i = 0; i < h->n_contexts; i++) + GNUNET_free (h->contexts[i]); + GNUNET_free (h->contexts); + } + GNUNET_free (h); +} + + +/* end of regex_internal_dht.c */ diff --git a/src/regex/regex_internal_lib.h b/src/regex/regex_internal_lib.h new file mode 100644 index 000000000..dc194546d --- /dev/null +++ b/src/regex/regex_internal_lib.h @@ -0,0 +1,267 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/regex_internal_lib.h + * @brief library to parse regular expressions into dfa + * @author Maximilian Szengel + */ + +#ifndef REGEX_INTERNAL_LIB_H +#define REGEX_INTERNAL_LIB_H + +#include "gnunet_util_lib.h" +#include "gnunet_dht_service.h" +#include "gnunet_statistics_service.h" +#include "regex_block_lib.h" + +#ifdef __cplusplus +extern "C" +{ +#if 0 /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + + +/** + * Automaton (NFA/DFA) representation. + */ +struct REGEX_INTERNAL_Automaton; + + +/** + * Construct DFA for the given 'regex' of length 'len'. + * + * Path compression means, that for example a DFA o -> a -> b -> c -> o will be + * compressed to o -> abc -> o. Note that this parameter influences the + * non-determinism of states of the resulting NFA in the DHT (number of outgoing + * edges with the same label). For example for an application that stores IPv4 + * addresses as bitstrings it could make sense to limit the path compression to + * 4 or 8. + * + * @param regex regular expression string. + * @param len length of the regular expression. + * @param max_path_len limit the path compression length to the + * given value. If set to 1, no path compression is applied. Set to 0 for + * maximal possible path compression (generally not desireable). + * @return DFA, needs to be freed using #REGEX_INTERNAL_automaton_destroy(). + */ +struct REGEX_INTERNAL_Automaton * +REGEX_INTERNAL_construct_dfa (const char *regex, + const size_t len, + unsigned int max_path_len); + + +/** + * Free the memory allocated by constructing the REGEX_INTERNAL_Automaton. + * data structure. + * + * @param a automaton to be destroyed. + */ +void +REGEX_INTERNAL_automaton_destroy (struct REGEX_INTERNAL_Automaton *a); + + +/** + * Evaluates the given 'string' against the given compiled regex. + * + * @param a automaton. + * @param string string to check. + * + * @return 0 if string matches, non 0 otherwise. + */ +int +REGEX_INTERNAL_eval (struct REGEX_INTERNAL_Automaton *a, + const char *string); + + +/** + * Get the first key for the given @a input_string. This hashes + * the first x bits of the @a input_string. + * + * @param input_string string. + * @param string_len length of the @a input_string. + * @param key pointer to where to write the hash code. + * @return number of bits of @a input_string that have been consumed + * to construct the key + */ +size_t +REGEX_INTERNAL_get_first_key (const char *input_string, + size_t string_len, + struct GNUNET_HashCode * key); + + +/** + * Iterator callback function. + * + * @param cls closure. + * @param key hash for current state. + * @param proof proof for current state + * @param accepting #GNUNET_YES if this is an accepting state, #GNUNET_NO if not. + * @param num_edges number of edges leaving current state. + * @param edges edges leaving current state. + */ +typedef void +(*REGEX_INTERNAL_KeyIterator)(void *cls, + const struct GNUNET_HashCode *key, + const char *proof, + int accepting, + unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges); + + +/** + * Iterate over all edges starting from start state of automaton 'a'. Calling + * iterator for each edge. + * + * @param a automaton. + * @param iterator iterator called for each edge. + * @param iterator_cls closure. + */ +void +REGEX_INTERNAL_iterate_all_edges (struct REGEX_INTERNAL_Automaton *a, + REGEX_INTERNAL_KeyIterator iterator, + void *iterator_cls); + + +/** + * Iterate over all edges of automaton 'a' that are reachable from a state with + * a proof of at least #GNUNET_REGEX_INITIAL_BYTES characters. + * + * Call the iterator for each such edge. + * + * @param a automaton. + * @param iterator iterator called for each reachable edge. + * @param iterator_cls closure. + */ +void +REGEX_INTERNAL_iterate_reachable_edges (struct REGEX_INTERNAL_Automaton *a, + REGEX_INTERNAL_KeyIterator iterator, + void *iterator_cls); + + + +/** + * Handle to store cached data about a regex announce. + */ +struct REGEX_INTERNAL_Announcement; + +/** + * Handle to store data about a regex search. + */ +struct REGEX_INTERNAL_Search; + + +/** + * Announce a regular expression: put all states of the automaton in the DHT. + * Does not free resources, must call #REGEX_INTERNAL_announce_cancel() for that. + * + * @param dht An existing and valid DHT service handle. CANNOT be NULL. + * @param priv our private key, must remain valid until the announcement is cancelled + * @param regex Regular expression to announce. + * @param compression How many characters per edge can we squeeze? + * @param stats Optional statistics handle to report usage. Can be NULL. + * @return Handle to reuse o free cached resources. + * Must be freed by calling #REGEX_INTERNAL_announce_cancel(). + */ +struct REGEX_INTERNAL_Announcement * +REGEX_INTERNAL_announce (struct GNUNET_DHT_Handle *dht, + const struct GNUNET_CRYPTO_EddsaPrivateKey *priv, + const char *regex, + uint16_t compression, + struct GNUNET_STATISTICS_Handle *stats); + + +/** + * Announce again a regular expression previously announced. + * Does use caching to speed up process. + * + * @param h Handle returned by a previous #REGEX_INTERNAL_announce() call. + */ +void +REGEX_INTERNAL_reannounce (struct REGEX_INTERNAL_Announcement *h); + + +/** + * Clear all cached data used by a regex announce. + * Does not close DHT connection. + * + * @param h Handle returned by a previous #REGEX_INTERNAL_announce() call. + */ +void +REGEX_INTERNAL_announce_cancel (struct REGEX_INTERNAL_Announcement *h); + + +/** + * Search callback function. + * + * @param cls Closure provided in #REGEX_INTERNAL_search(). + * @param id Peer providing a regex that matches the string. + * @param get_path Path of the get request. + * @param get_path_length Length of @a get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the @a put_path. + */ +typedef void +(*REGEX_INTERNAL_Found)(void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length); + + +/** + * Search for a peer offering a regex matching certain string in the DHT. + * The search runs until #REGEX_INTERNAL_search_cancel() is called, even if results + * are returned. + * + * @param dht An existing and valid DHT service handle. + * @param string String to match against the regexes in the DHT. + * @param callback Callback for found peers. + * @param callback_cls Closure for @c callback. + * @param stats Optional statistics handle to report usage. Can be NULL. + * @return Handle to stop search and free resources. + * Must be freed by calling #REGEX_INTERNAL_search_cancel(). + */ +struct REGEX_INTERNAL_Search * +REGEX_INTERNAL_search (struct GNUNET_DHT_Handle *dht, + const char *string, + REGEX_INTERNAL_Found callback, + void *callback_cls, + struct GNUNET_STATISTICS_Handle *stats); + +/** + * Stop search and free all data used by a #REGEX_INTERNAL_search() call. + * Does not close DHT connection. + * + * @param h Handle returned by a previous #REGEX_INTERNAL_search() call. + */ +void +REGEX_INTERNAL_search_cancel (struct REGEX_INTERNAL_Search *h); + + +#if 0 /* keep Emacsens' auto-indent happy */ +{ +#endif +#ifdef __cplusplus +} +#endif + +/* end of regex_internal_lib.h */ +#endif diff --git a/src/regex/regex_ipc.h b/src/regex/regex_ipc.h new file mode 100644 index 000000000..71ac273cd --- /dev/null +++ b/src/regex/regex_ipc.h @@ -0,0 +1,105 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/regex_ipc.h + * @brief regex IPC messages (not called 'regex.h' due to conflict with + * system headers) + * @author Christian Grothoff + */ +#ifndef REGEX_IPC_H +#define REGEX_IPC_H + +#include "gnunet_util_lib.h" + +/** + * Request for regex service to announce capability. + */ +struct AnnounceMessage +{ + + /** + * Type is GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE + */ + struct GNUNET_MessageHeader header; + + /** + * How many characters can we squeeze per edge? + */ + uint16_t compression; + + /** + * Always zero. + */ + uint16_t reserved; + + /** + * Delay between repeated announcements. + */ + struct GNUNET_TIME_RelativeNBO refresh_delay; + + /* followed by 0-terminated regex as string */ +}; + + +/** + * Message to initiate regex search. + */ +struct RegexSearchMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_REGEX_SEARCH + */ + struct GNUNET_MessageHeader header; + + /* followed by 0-terminated search string */ + +}; + + +/** + * Result from regex search. + */ +struct ResultMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_REGEX_RESULT + */ + struct GNUNET_MessageHeader header; + + /** + * Number of entries in the GET path. + */ + uint16_t get_path_length; + + /** + * Number of entries in the PUT path. + */ + uint16_t put_path_length; + + /** + * Identity of the peer that was found. + */ + struct GNUNET_PeerIdentity id; + + /* followed by GET path and PUT path arrays */ + +}; + + +/* end of regex_ipc.h */ +#endif diff --git a/src/regex/regex_simulation_profiler_test.conf b/src/regex/regex_simulation_profiler_test.conf new file mode 100644 index 000000000..9384aa249 --- /dev/null +++ b/src/regex/regex_simulation_profiler_test.conf @@ -0,0 +1,7 @@ +[regex-mysql] +DATABASE = regex +USER = gnunet +PASSWORD = +HOST = localhost +PORT = 3306 +REGEX_PREFIX = GNVPN-0001-PAD diff --git a/src/regex/regex_test_graph.c b/src/regex/regex_test_graph.c new file mode 100644 index 000000000..d8f16e894 --- /dev/null +++ b/src/regex/regex_test_graph.c @@ -0,0 +1,316 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file src/regex/regex_test_graph.c + * @brief functions for creating .dot graphs from regexes + * @author Maximilian Szengel + */ +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" +#include "regex_internal.h" + +/** + * Context for graph creation. Passed as the cls to + * REGEX_TEST_automaton_save_graph_step. + */ +struct REGEX_TEST_Graph_Context +{ + /** + * File pointer to the dot file used for output. + */ + FILE *filep; + + /** + * Verbose flag, if it's set to GNUNET_YES additional info will be printed in + * the graph. + */ + int verbose; + + /** + * Coloring flag, if set to GNUNET_YES SCCs will be colored. + */ + int coloring; +}; + + +/** + * Recursive function doing DFS with 'v' as a start, detecting all SCCs inside + * the subgraph reachable from 'v'. Used with scc_tarjan function to detect all + * SCCs inside an automaton. + * + * @param scc_counter counter for numbering the sccs + * @param v start vertex + * @param index current index + * @param stack stack for saving all SCCs + * @param stack_size current size of the stack + */ +static void +scc_tarjan_strongconnect (unsigned int *scc_counter, + struct REGEX_INTERNAL_State *v, unsigned int *index, + struct REGEX_INTERNAL_State **stack, + unsigned int *stack_size) +{ + struct REGEX_INTERNAL_State *w; + struct REGEX_INTERNAL_Transition *t; + + v->index = *index; + v->lowlink = *index; + (*index)++; + stack[(*stack_size)++] = v; + v->contained = 1; + + for (t = v->transitions_head; NULL != t; t = t->next) + { + w = t->to_state; + + if (NULL == w) + continue; + + if (w->index < 0) + { + scc_tarjan_strongconnect (scc_counter, w, index, stack, stack_size); + v->lowlink = (v->lowlink > w->lowlink) ? w->lowlink : v->lowlink; + } + else if (1 == w->contained) + v->lowlink = (v->lowlink > w->index) ? w->index : v->lowlink; + } + + if (v->lowlink == v->index) + { + (*scc_counter)++; + do + { + w = stack[--(*stack_size)]; + w->contained = 0; + w->scc_id = *scc_counter; + } + while (w != v); + } +} + + +/** + * Detect all SCCs (Strongly Connected Components) inside the given automaton. + * SCCs will be marked using the scc_id on each state. + * + * @param a the automaton for which SCCs should be computed and assigned. + */ +static void +scc_tarjan (struct REGEX_INTERNAL_Automaton *a) +{ + unsigned int index; + unsigned int scc_counter; + struct REGEX_INTERNAL_State *v; + struct REGEX_INTERNAL_State *stack[a->state_count]; + unsigned int stack_size; + + for (v = a->states_head; NULL != v; v = v->next) + { + v->contained = 0; + v->index = -1; + v->lowlink = -1; + } + + stack_size = 0; + index = 0; + scc_counter = 0; + + for (v = a->states_head; NULL != v; v = v->next) + { + if (v->index < 0) + scc_tarjan_strongconnect (&scc_counter, v, &index, stack, &stack_size); + } +} + + +/** + * Save a state to an open file pointer. cls is expected to be a file pointer to + * an open file. Used only in conjunction with + * REGEX_TEST_automaton_save_graph. + * + * @param cls file pointer. + * @param count current count of the state, not used. + * @param s state. + */ +void +REGEX_TEST_automaton_save_graph_step (void *cls, unsigned int count, + struct REGEX_INTERNAL_State *s) +{ + struct REGEX_TEST_Graph_Context *ctx = cls; + struct REGEX_INTERNAL_Transition *ctran; + char *s_acc = NULL; + char *s_tran = NULL; + char *name; + char *to_name; + + if (GNUNET_YES == ctx->verbose) + GNUNET_asprintf (&name, "%i (%s) (%s) (%s)", s->dfs_id, s->name, s->proof, + GNUNET_h2s (&s->hash)); + else + GNUNET_asprintf (&name, "%i", s->dfs_id); + + if (s->accepting) + { + if (GNUNET_YES == ctx->coloring) + { + GNUNET_asprintf (&s_acc, + "\"%s\" [shape=doublecircle, color=\"0.%i 0.8 0.95\"];\n", + name, s->scc_id * s->scc_id); + } + else + { + GNUNET_asprintf (&s_acc, "\"%s\" [shape=doublecircle];\n", name, + s->scc_id); + } + } + else if (GNUNET_YES == ctx->coloring) + { + GNUNET_asprintf (&s_acc, + "\"%s\" [shape=circle, color=\"0.%i 0.8 0.95\"];\n", name, + s->scc_id * s->scc_id); + } + else + { + GNUNET_asprintf (&s_acc, "\"%s\" [shape=circle];\n", name, s->scc_id); + } + + GNUNET_assert (NULL != s_acc); + + fwrite (s_acc, strlen (s_acc), 1, ctx->filep); + GNUNET_free (s_acc); + s_acc = NULL; + + for (ctran = s->transitions_head; NULL != ctran; ctran = ctran->next) + { + if (NULL == ctran->to_state) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Transition from State %i has no state for transitioning\n", + s->id); + continue; + } + + if (GNUNET_YES == ctx->verbose) + { + GNUNET_asprintf (&to_name, "%i (%s) (%s) (%s)", ctran->to_state->dfs_id, + ctran->to_state->name, ctran->to_state->proof, + GNUNET_h2s (&ctran->to_state->hash)); + } + else + GNUNET_asprintf (&to_name, "%i", ctran->to_state->dfs_id); + + if (NULL == ctran->label) + { + if (GNUNET_YES == ctx->coloring) + { + GNUNET_asprintf (&s_tran, + "\"%s\" -> \"%s\" [label = \"ε\", color=\"0.%i 0.8 0.95\"];\n", + name, to_name, s->scc_id * s->scc_id); + } + else + { + GNUNET_asprintf (&s_tran, "\"%s\" -> \"%s\" [label = \"ε\"];\n", name, + to_name, s->scc_id); + } + } + else + { + if (GNUNET_YES == ctx->coloring) + { + GNUNET_asprintf (&s_tran, + "\"%s\" -> \"%s\" [label = \"%s\", color=\"0.%i 0.8 0.95\"];\n", + name, to_name, ctran->label, s->scc_id * s->scc_id); + } + else + { + GNUNET_asprintf (&s_tran, "\"%s\" -> \"%s\" [label = \"%s\"];\n", name, + to_name, ctran->label, s->scc_id); + } + } + + GNUNET_free (to_name); + + GNUNET_assert (NULL != s_tran); + + fwrite (s_tran, strlen (s_tran), 1, ctx->filep); + GNUNET_free (s_tran); + s_tran = NULL; + } + + GNUNET_free (name); +} + + +/** + * Save the given automaton as a GraphViz dot file. + * + * @param a the automaton to be saved. + * @param filename where to save the file. + * @param options options for graph generation that include coloring or verbose + * mode + */ +void +REGEX_TEST_automaton_save_graph (struct REGEX_INTERNAL_Automaton *a, + const char *filename, + enum REGEX_TEST_GraphSavingOptions options) +{ + char *start; + char *end; + struct REGEX_TEST_Graph_Context ctx; + + if (NULL == a) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not print NFA, was NULL!"); + return; + } + + if (NULL == filename || strlen (filename) < 1) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No Filename given!"); + return; + } + + ctx.filep = fopen (filename, "w"); + ctx.verbose = + (0 == (options & REGEX_TEST_GRAPH_VERBOSE)) ? GNUNET_NO : GNUNET_YES; + ctx.coloring = + (0 == (options & REGEX_TEST_GRAPH_COLORING)) ? GNUNET_NO : GNUNET_YES; + + if (NULL == ctx.filep) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not open file for writing: %s", + filename); + return; + } + + /* First add the SCCs to the automaton, so we can color them nicely */ + if (GNUNET_YES == ctx.coloring) + scc_tarjan (a); + + start = "digraph G {\nrankdir=LR\n"; + fwrite (start, strlen (start), 1, ctx.filep); + + REGEX_INTERNAL_automaton_traverse (a, a->start, NULL, NULL, + ®EX_TEST_automaton_save_graph_step, + &ctx); + + end = "\n}\n"; + fwrite (end, strlen (end), 1, ctx.filep); + fclose (ctx.filep); +} diff --git a/src/regex/regex_test_lib.c b/src/regex/regex_test_lib.c new file mode 100644 index 000000000..d25799ea4 --- /dev/null +++ b/src/regex/regex_test_lib.c @@ -0,0 +1,646 @@ +/* + * This file is part of GNUnet + * Copyright (C) 2012-2017 GNUnet e.V. + * + * GNUnet is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * GNUnet is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +/** + * @file src/regex/regex_test_lib.c + * @brief library to read regexes representing IP networks from a file. + * and simplyfinying the into one big regex, in order to run + * tests (regex performance, cadet profiler). + * @author Bartlomiej Polot + */ + +#include "platform.h" +#include "gnunet_util_lib.h" + + +/** + * Struct to hold the tree formed by prefix-combining the regexes. + */ +struct RegexCombineCtx +{ + /** + * Child nodes with same prefix and token. + */ + struct RegexCombineCtx **children; + + /** + * Alphabet size (how many @a children there are) + */ + unsigned int size; + + /** + * Token. + */ + char *s; +}; + + +/** + * Char 2 int + * + * Convert a character into its int value depending on the base used + * + * @param c Char + * @param size base (2, 8 or 16(hex)) + * + * @return Int in range [0, (base-1)] + */ +static int +c2i (char c, int size) +{ + switch (size) + { + case 2: + case 8: + return c - '0'; + break; + case 16: + if (c >= '0' && c <= '9') + return c - '0'; + else if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + else if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + else + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Cannot convert char %c in base %u\n", + c, size); + GNUNET_assert (0); + } + break; + default: + GNUNET_assert (0); + } +} + + +/** + * Printf spaces to indent the regex tree + * + * @param n Indentation level + */ +static void +space (int n) +{ + for (int i = 0; i < n; i++) + fprintf (stderr, "| "); +} + + +/** + * Printf the combined regex ctx. + * + * @param ctx The ctx to printf + * @param level Indentation level to start with + */ +static void +debugctx (struct RegexCombineCtx *ctx, int level) +{ +#if DEBUG_REGEX + if (NULL != ctx->s) + { + space (level - 1); + fprintf (stderr, "%u:'%s'\n", c2i(ctx->s[0], ctx->size), ctx->s); + } + else + fprintf (stderr, "ROOT (base %u)\n", ctx->size); + for (unsigned int i = 0; i < ctx->size; i++) + { + if (NULL != ctx->children[i]) + { + space (level); + debugctx (ctx->children[i], level + 1); + } + } + fflush(stderr); +#endif +} + + +/** + * Add a single regex to a context, combining with exisiting regex by-prefix. + * + * @param ctx Context with 0 or more regexes. + * @param regex Regex to add. + */ +static void +regex_add (struct RegexCombineCtx *ctx, + const char *regex); + + +/** + * Create and initialize a new RegexCombineCtx. + * + * @param alphabet_size Size of the alphabet (and the Trie array) + */ +static struct RegexCombineCtx * +new_regex_ctx (unsigned int alphabet_size) +{ + struct RegexCombineCtx *ctx; + size_t array_size; + + array_size = sizeof(struct RegexCombineCtx *) * alphabet_size; + ctx = GNUNET_new (struct RegexCombineCtx); + ctx->children = GNUNET_malloc (array_size); + ctx->size = alphabet_size; + + return ctx; +} + + +static void +move_children (struct RegexCombineCtx *dst, + const struct RegexCombineCtx *src) +{ + size_t array_size; + + array_size = sizeof(struct RegexCombineCtx *) * src->size; + GNUNET_memcpy (dst->children, + src->children, + array_size); + for (unsigned int i = 0; i < src->size; i++) + { + src->children[i] = NULL; + } +} + + +/** + * Extract a string from all prefix-combined regexes. + * + * @param ctx Context with 0 or more regexes. + * + * @return Regex that matches any of the added regexes. + */ +static char * +regex_combine (struct RegexCombineCtx *ctx) +{ + struct RegexCombineCtx *p; + unsigned int i; + size_t len; + char *regex; + char *tmp; + char *s; + int opt; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "new combine %s\n", ctx->s); + regex = GNUNET_strdup (""); + opt = GNUNET_NO; + for (i = 0; i < ctx->size; i++) + { + p = ctx->children[i]; + if (NULL == p) + continue; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "adding '%s' to innner %s\n", + p->s, ctx->s); + s = regex_combine (p); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " total '%s'\n", s); + if (strlen(s) == 0) + { + opt = GNUNET_YES; + } + else + { + GNUNET_asprintf (&tmp, "%s%s|", regex, s); + GNUNET_free_non_null (regex); + regex = tmp; + } + GNUNET_free_non_null (s); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " so far '%s' for inner %s\n", regex, ctx->s); + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "opt: %d, innner: '%s'\n", opt, regex); + len = strlen (regex); + if (0 == len) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "empty, returning ''\n"); + GNUNET_free (regex); + return NULL == ctx->s ? NULL : GNUNET_strdup (ctx->s); + } + + if ('|' == regex[len - 1]) + regex[len - 1] = '\0'; + + if (NULL != ctx->s) + { + if (opt) + GNUNET_asprintf (&s, "%s(%s)?", ctx->s, regex); + else + GNUNET_asprintf (&s, "%s(%s)", ctx->s, regex); + GNUNET_free (regex); + regex = s; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "partial: %s\n", regex); + return regex; +} + + +/** + * Get the number of matching characters on the prefix of both strings. + * + * @param s1 String 1. + * @param s2 String 2. + * + * @return Number of characters of matching prefix. + */ +static unsigned int +get_prefix_length (const char *s1, const char *s2) +{ + unsigned int l1; + unsigned int l2; + unsigned int limit; + unsigned int i; + + l1 = strlen (s1); + l2 = strlen (s2); + limit = l1 > l2 ? l2 : l1; + + for (i = 0; i < limit; i++) + { + if (s1[i] != s2[i]) + return i; + } + return limit; +} + + +/** + * Return the child context with the longest prefix match with the regex. + * Usually only one child will match, search all just in case. + * + * @param ctx Context whose children to search. + * @param regex String to match. + * + * @return Child with the longest prefix, NULL if no child matches. + */ +static struct RegexCombineCtx * +get_longest_prefix (struct RegexCombineCtx *ctx, const char *regex) +{ + struct RegexCombineCtx *p; + struct RegexCombineCtx *best; + unsigned int i; + unsigned int l; + unsigned int best_l; + + best_l = 0; + best = NULL; + + for (i = 0; i < ctx->size; i++) + { + p = ctx->children[i]; + if (NULL == p) + continue; + + l = get_prefix_length (p->s, regex); + if (l > best_l) + { + GNUNET_break (0 == best_l); + best = p; + best_l = l; + } + } + return best; +} + +static void +regex_add_multiple (struct RegexCombineCtx *ctx, + const char *regex, + struct RegexCombineCtx **children) +{ + char tmp[2]; + long unsigned int i; + size_t l; + struct RegexCombineCtx *newctx; + unsigned int count; + + if ('(' != regex[0]) + { + GNUNET_assert (0); + } + + /* Does the regex cover *all* possible children? Then don't add any, + * as it will be covered by the post-regex "(a-z)*" + */ + l = strlen (regex); + count = 0; + for (i = 1UL; i < l; i++) + { + if (regex[i] != '|' && regex[i] != ')') + { + count++; + } + } + if (count == ctx->size) + { + return; + } + + /* Add every component as a child node */ + tmp[1] = '\0'; + for (i = 1UL; i < l; i++) + { + if (regex[i] != '|' && regex[i] != ')') + { + tmp[0] = regex[i]; + newctx = new_regex_ctx(ctx->size); + newctx->s = GNUNET_strdup (tmp); + if (children != NULL) + GNUNET_memcpy (newctx->children, + children, + sizeof (*children) * ctx->size); + ctx->children[c2i(tmp[0], ctx->size)] = newctx; + } + } +} + +/** + * Add a single regex to a context, splitting the exisiting state. + * + * We only had a partial match, split existing state, truncate the current node + * so it only contains the prefix, add suffix(es) as children. + * + * @param ctx Context to split. + * @param len Lenght of ctx->s + * @param prefix_l Lenght of common prefix of the new regex and @a ctx->s + */ +static void +regex_split (struct RegexCombineCtx *ctx, + unsigned int len, + unsigned int prefix_l) +{ + struct RegexCombineCtx *newctx; + unsigned int idx; + char *suffix; + + suffix = GNUNET_malloc (len - prefix_l + 1); + strncpy (suffix, &ctx->s[prefix_l], len - prefix_l + 1); + + /* Suffix saved, truncate current node so it only contains the prefix, + * copy any children nodes to put as grandchildren and initialize new empty + * children array. + */ + ctx->s[prefix_l] = '\0'; + + /* If the suffix is an OR expression, add multiple children */ + if ('(' == suffix[0]) + { + struct RegexCombineCtx **tmp; + + tmp = ctx->children; + ctx->children = GNUNET_malloc (sizeof(*tmp) * ctx->size); + regex_add_multiple (ctx, suffix, tmp); + GNUNET_free (suffix); + GNUNET_free (tmp); + return; + } + + /* The suffix is a normal string, add as one node */ + newctx = new_regex_ctx (ctx->size); + newctx->s = suffix; + move_children (newctx, ctx); + idx = c2i(suffix[0], ctx->size); + ctx->children[idx] = newctx; +} + + +/** + * Add a single regex to a context, combining with exisiting regex by-prefix. + * + * @param ctx Context with 0 or more regexes. + * @param regex Regex to add. + */ +static void +regex_add (struct RegexCombineCtx *ctx, const char *regex) +{ + struct RegexCombineCtx *p; + struct RegexCombineCtx *newctx; + long unsigned int l; + unsigned int prefix_l; + const char *rest_r; + const char *rest_s; + size_t len; + int idx; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "regex_add '%s' into '%s'\n", + regex, ctx->s); + l = strlen (regex); + if (0UL == l) + return; + + /* If the regex is in the form of (a|b|c), add every character separately */ + if ('(' == regex[0]) + { + regex_add_multiple (ctx, regex, NULL); + return; + } + + p = get_longest_prefix (ctx, regex); + if (NULL != p) + { + /* There is some prefix match, reduce regex and try again */ + prefix_l = get_prefix_length (p->s, regex); + rest_s = &p->s[prefix_l]; + rest_r = ®ex[prefix_l]; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "chosen '%s' [%u]\n", p->s, prefix_l); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "prefix r '%.*s'\n", prefix_l, p->s); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "rest r '%s'\n", rest_r); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "rest s '%s'\n", rest_s); + len = strlen (p->s); + if (prefix_l < len) + { + regex_split (p, len, prefix_l); + } + regex_add (p, rest_r); + return; + } + + /* There is no prefix match, add new */ + idx = c2i(regex[0], ctx->size); + if (NULL == ctx->children[idx] && NULL != ctx->s) + { + /* this was the end before, add empty string */ + newctx = new_regex_ctx (ctx->size); + newctx->s = GNUNET_strdup (""); + ctx->children[idx] = newctx; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " no match\n"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " new state %s\n", regex); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " under %s\n", ctx->s); + newctx = new_regex_ctx(ctx->size); + newctx->s = GNUNET_strdup (regex); + ctx->children[idx] = newctx; +} + + +/** + * Free all resources used by the context node and all its children. + * + * @param ctx Context to free. + */ +static void +regex_ctx_destroy (struct RegexCombineCtx *ctx) +{ + unsigned int i; + + if (NULL == ctx) + return; + + for (i = 0; i < ctx->size; i++) + { + regex_ctx_destroy (ctx->children[i]); + } + GNUNET_free_non_null (ctx->s); /* 's' on root node is null */ + GNUNET_free (ctx->children); + GNUNET_free (ctx); +} + + +/** + * Combine an array of regexes into a single prefix-shared regex. + * Returns a prefix-combine regex that matches the same strings as + * any of the original regexes. + * + * WARNING: only useful for reading specific regexes for specific applications, + * namely the gnunet-regex-profiler / gnunet-regex-daemon. + * This function DOES NOT support arbitrary regex combining. + * + * @param regexes A NULL-terminated array of regexes. + * @param alphabet_size Size of the alphabet the regex uses. + * + * @return A string with a single regex that matches any of the original regexes + */ +char * +REGEX_TEST_combine (char * const regexes[], unsigned int alphabet_size) +{ + unsigned int i; + char *combined; + const char *current; + struct RegexCombineCtx *ctx; + + ctx = new_regex_ctx (alphabet_size); + for (i = 0; regexes[i]; i++) + { + current = regexes[i]; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Regex %u: %s\n", i, current); + regex_add (ctx, current); + debugctx (ctx, 0); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "\nCombining...\n"); + debugctx (ctx, 0); + + combined = regex_combine (ctx); + + regex_ctx_destroy (ctx); + + return combined; +} + + +/** + * Read a set of regexes from a file, one per line and return them in an array + * suitable for REGEX_TEST_combine. + * The array must be free'd using REGEX_TEST_free_from_file. + * + * @param filename Name of the file containing the regexes. + * + * @return A newly allocated, NULL terminated array of regexes. + */ +char ** +REGEX_TEST_read_from_file (const char *filename) +{ + struct GNUNET_DISK_FileHandle *f; + unsigned int nr; + unsigned int offset; + off_t size; + size_t len; + char *buffer; + char *regex; + char **regexes; + + f = GNUNET_DISK_file_open (filename, + GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (NULL == f) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Can't open file %s for reading\n", filename); + return NULL; + } + if (GNUNET_OK != GNUNET_DISK_file_handle_size (f, &size)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Can't get size of file %s\n", filename); + GNUNET_DISK_file_close (f); + return NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "using file %s, size %llu\n", + filename, (unsigned long long) size); + + buffer = GNUNET_malloc (size + 1); + GNUNET_DISK_file_read (f, buffer, size); + GNUNET_DISK_file_close (f); + regexes = GNUNET_malloc (sizeof (char *)); + nr = 1; + offset = 0; + regex = NULL; + do + { + if (NULL == regex) + regex = GNUNET_malloc (size + 1); + len = (size_t) sscanf (&buffer[offset], "%s", regex); + if (0 == len) + break; + len = strlen (regex); + offset += len + 1; + if (len < 1) + continue; + regex[len] = '\0'; + regex = GNUNET_realloc (regex, len + 1); + GNUNET_array_grow (regexes, nr, nr + 1); + GNUNET_assert (NULL == regexes[nr - 2]); + regexes[nr - 2] = regex; + regexes[nr - 1] = NULL; + regex = NULL; + } while (offset < size); + GNUNET_free_non_null (regex); + GNUNET_free (buffer); + + return regexes; +} + + +/** + * Free all memory reserved for a set of regexes created by read_from_file. + * + * @param regexes NULL-terminated array of regexes. + */ +void +REGEX_TEST_free_from_file (char **regexes) +{ + unsigned int i; + + for (i = 0; regexes[i]; i++) + GNUNET_free (regexes[i]); + GNUNET_free (regexes); +} + +/* end of regex_test_lib.c */ diff --git a/src/regex/regex_test_lib.h b/src/regex/regex_test_lib.h new file mode 100644 index 000000000..533ba3efb --- /dev/null +++ b/src/regex/regex_test_lib.h @@ -0,0 +1,157 @@ +/* + * This file is part of GNUnet + * Copyright (C) 2012 GNUnet e.V. + * + * GNUnet is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * GNUnet is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +/** + * @file src/regex/regex_test_lib.h + * @brief library to read regexes representing IP networks from a file. + * and simplifying the into one big regex, in order to run + * tests (regex performance, regex profiler). + * @author Bertlomiej Polot + */ + +#ifndef REGEX_INTERNAL_TEST_LIB_H +#define REGEX_INTERNAL_TEST_LIB_H + +#include "regex_internal_lib.h" + +#ifdef __cplusplus +extern "C" +{ + #if 0 /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + + +/** + * Combine an array of regexes into a single prefix-shared regex. + * Returns a prefix-combine regex that matches the same strings as + * any of the original regexes. + * + * WARNING: only useful for reading specific regexes for specific applications, + * namely the gnunet-regex-profiler / gnunet-regex-daemon. + * This function DOES NOT support arbitrary regex combining. + * + * @param regexes A NULL-terminated array of regexes. + * @param alphabet_size Size of the alphabet the regex uses. + * + * @return A string with a single regex that matches any of the original regexes + */ +char * +REGEX_TEST_combine (char * const regexes[], unsigned int alphabet_size); + + +/** + * Read a set of regexes from a file, one per line and return them in an array + * suitable for REGEX_TEST_combine. + * The array must be free'd using REGEX_TEST_free_from_file. + * + * @param filename Name of the file containing the regexes. + * + * @return A newly allocated, NULL terminated array of regexes. + */ +char ** +REGEX_TEST_read_from_file (const char *filename); + + +/** + * Free all memory reserved for a set of regexes created by read_from_file. + * + * @param regexes NULL-terminated array of regexes. + */ +void +REGEX_TEST_free_from_file (char **regexes); + + +/** + * Generate a (pseudo) random regular expression of length 'rx_length', as well + * as a (optional) string that will be matched by the generated regex. The + * returned regex needs to be freed. + * + * @param rx_length length of the random regex. + * @param matching_str (optional) pointer to a string that will contain a string + * that will be matched by the generated regex, if + * 'matching_str' pointer was not NULL. + * + * @return NULL if 'rx_length' is 0, a random regex of length 'rx_length', which + * needs to be freed, otherwise. + */ +char * +REGEX_TEST_generate_random_regex (size_t rx_length, char *matching_str); + + +/** + * Generate a random string of maximum length 'max_len' that only contains literals allowed + * in a regular expression. The string might be 0 chars long but is garantueed + * to be shorter or equal to 'max_len'. + * + * @param max_len maximum length of the string that should be generated. + * + * @return random string that needs to be freed. + */ +char * +REGEX_TEST_generate_random_string (size_t max_len); + + +/** + * Options for graph creation function + * REGEX_TEST_automaton_save_graph. + */ +enum REGEX_TEST_GraphSavingOptions +{ + /** + * Default. Do nothing special. + */ + REGEX_TEST_GRAPH_DEFAULT = 0, + + /** + * The generated graph will include extra information such as the NFA states + * that were used to generate the DFA state. + */ + REGEX_TEST_GRAPH_VERBOSE = 1, + + /** + * Enable graph coloring. Will color each SCC in a different color. + */ + REGEX_TEST_GRAPH_COLORING = 2 +}; + + +/** + * Save the given automaton as a GraphViz dot file. + * + * @param a the automaton to be saved. + * @param filename where to save the file. + * @param options options for graph generation that include coloring or verbose + * mode + */ +void +REGEX_TEST_automaton_save_graph (struct REGEX_INTERNAL_Automaton *a, + const char *filename, + enum REGEX_TEST_GraphSavingOptions options); + + + +#if 0 /* keep Emacsens' auto-indent happy */ +{ + #endif + #ifdef __cplusplus +} +#endif + +/* end of regex_internal_lib.h */ +#endif diff --git a/src/regex/regex_test_random.c b/src/regex/regex_test_random.c new file mode 100644 index 000000000..6e51885ee --- /dev/null +++ b/src/regex/regex_test_random.c @@ -0,0 +1,169 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file src/regex/regex_test_random.c + * @brief functions for creating random regular expressions and strings + * @author Maximilian Szengel + */ +#include "platform.h" +#include "regex_test_lib.h" +#include "gnunet_crypto_lib.h" +#include "regex_internal.h" + + +/** + * Get a (pseudo) random valid literal for building a regular expression. + * + * @return random valid literal + */ +static char +get_random_literal () +{ + uint32_t ridx; + + ridx = + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + (uint32_t) strlen (ALLOWED_LITERALS)); + + return ALLOWED_LITERALS[ridx]; +} + + +/** + * Generate a (pseudo) random regular expression of length 'rx_length', as well + * as a (optional) string that will be matched by the generated regex. The + * returned regex needs to be freed. + * + * @param rx_length length of the random regex. + * @param matching_str (optional) pointer to a string that will contain a string + * that will be matched by the generated regex, if + * 'matching_str' pointer was not NULL. Make sure you + * allocated at least rx_length+1 bytes for this sting. + * + * @return NULL if 'rx_length' is 0, a random regex of length 'rx_length', which + * needs to be freed, otherwise. + */ +char * +REGEX_TEST_generate_random_regex (size_t rx_length, char *matching_str) +{ + char *rx; + char *rx_p; + char *matching_strp; + unsigned int i; + unsigned int char_op_switch; + unsigned int last_was_op; + int rx_op; + char current_char; + + if (0 == rx_length) + return NULL; + + if (NULL != matching_str) + matching_strp = matching_str; + else + matching_strp = NULL; + + rx = GNUNET_malloc (rx_length + 1); + rx_p = rx; + current_char = 0; + last_was_op = 1; + + for (i = 0; i < rx_length; i++) + { + char_op_switch = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2); + + if (0 == char_op_switch && !last_was_op) + { + last_was_op = 1; + rx_op = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 4); + + switch (rx_op) + { + case 0: + current_char = '+'; + break; + case 1: + current_char = '*'; + break; + case 2: + current_char = '?'; + break; + case 3: + if (i < rx_length - 1) /* '|' cannot be at the end */ + current_char = '|'; + else + current_char = get_random_literal (); + break; + } + } + else + { + current_char = get_random_literal (); + last_was_op = 0; + } + + if (NULL != matching_strp && + (current_char != '+' && current_char != '*' && current_char != '?' && + current_char != '|')) + { + *matching_strp = current_char; + matching_strp++; + } + + *rx_p = current_char; + rx_p++; + } + *rx_p = '\0'; + if (NULL != matching_strp) + *matching_strp = '\0'; + + return rx; +} + + +/** + * Generate a random string of maximum length 'max_len' that only contains literals allowed + * in a regular expression. The string might be 0 chars long but is garantueed + * to be shorter or equal to 'max_len'. + * + * @param max_len maximum length of the string that should be generated. + * + * @return random string that needs to be freed. + */ +char * +REGEX_TEST_generate_random_string (size_t max_len) +{ + unsigned int i; + char *str; + size_t len; + + if (1 > max_len) + return GNUNET_strdup (""); + + len = (size_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, max_len); + str = GNUNET_malloc (len + 1); + + for (i = 0; i < len; i++) + { + str[i] = get_random_literal (); + } + + str[i] = '\0'; + + return str; +} diff --git a/src/regex/test_regex_api.c b/src/regex/test_regex_api.c new file mode 100644 index 000000000..a08e2ed6d --- /dev/null +++ b/src/regex/test_regex_api.c @@ -0,0 +1,127 @@ +/* + This file is part of GNUnet. + Copyright (C) 2013 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_api.c + * @brief base test case for regex api (and DHT functions) + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_testing_lib.h" +#include "gnunet_regex_service.h" + + +/** + * How long until we really give up on a particular testcase portion? + */ +#define TOTAL_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 600) + +/** + * How long until we give up on any particular operation (and retry)? + */ +#define BASE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3) + + +static struct GNUNET_REGEX_Announcement *a; + +static struct GNUNET_REGEX_Search *s; + +static int ok = 1; + +static struct GNUNET_SCHEDULER_Task *die_task; + + +static void +end (void *cls) +{ + die_task = NULL; + GNUNET_REGEX_announce_cancel (a); + a = NULL; + GNUNET_REGEX_search_cancel (s); + s = NULL; + ok = 0; +} + + +static void +end_badly () +{ + die_task = NULL; + FPRINTF (stderr, "%s", "Testcase failed (timeout).\n"); + GNUNET_REGEX_announce_cancel (a); + a = NULL; + GNUNET_REGEX_search_cancel (s); + s = NULL; + ok = 1; +} + + +/** + * Search callback function, invoked for every result that was found. + * + * @param cls Closure provided in GNUNET_REGEX_search. + * @param id Peer providing a regex that matches the string. + * @param get_path Path of the get request. + * @param get_path_length Lenght of get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the put_path. + */ +static void +found_cb (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length) +{ + GNUNET_SCHEDULER_cancel (die_task); + die_task = + GNUNET_SCHEDULER_add_now (&end, NULL); +} + + +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Peer *peer) +{ + die_task = + GNUNET_SCHEDULER_add_delayed (TOTAL_TIMEOUT, + &end_badly, NULL); + a = GNUNET_REGEX_announce (cfg, + "my long prefix - hello world(0|1)*", + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 5), + 1); + s = GNUNET_REGEX_search (cfg, + "my long prefix - hello world0101", + &found_cb, NULL); +} + + +int +main (int argc, char *argv[]) +{ + if (0 != GNUNET_TESTING_peer_run ("test-regex-api", + "test_regex_api_data.conf", + &run, NULL)) + return 1; + return ok; +} + +/* end of test_regex_api.c */ diff --git a/src/regex/test_regex_api_data.conf b/src/regex/test_regex_api_data.conf new file mode 100644 index 000000000..40fee1e54 --- /dev/null +++ b/src/regex/test_regex_api_data.conf @@ -0,0 +1,39 @@ +@INLINE@ ../../contrib/conf/gnunet/no_forcestart.conf +@INLINE@ ../../contrib/conf/gnunet/no_autostart_above_core.conf + +[PATHS] +GNUNET_TEST_HOME = $GNUNET_TMP/test-regex-api/ + +[dhtcache] +QUOTA = 1 MB +DATABASE = heap + +[topology] +TARGET-CONNECTION-COUNT = 16 +AUTOCONNECT = YES +FRIENDS-ONLY = NO +MINIMUM-FRIENDS = 0 + +[ats] +WAN_QUOTA_IN = 1 GB +WAN_QUOTA_OUT = 1 GB + +[dht] +START_ON_DEMAND = YES +PORT = 12370 + +[regex] +START_ON_DEMAND = YES + +[transport] +plugins = tcp +NEIGHBOUR_LIMIT = 50 + +[nat] +DISABLEV6 = YES +BINDTO = 127.0.0.1 +ENABLE_UPNP = NO +BEHIND_NAT = NO +ALLOW_NAT = NO +INTERNAL_ADDRESS = 127.0.0.1 +EXTERNAL_ADDRESS = 127.0.0.1 diff --git a/src/regex/test_regex_eval_api.c b/src/regex/test_regex_eval_api.c new file mode 100644 index 000000000..8a0c0d024 --- /dev/null +++ b/src/regex/test_regex_eval_api.c @@ -0,0 +1,364 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_eval_api.c + * @brief test for regex.c + * @author Maximilian Szengel + */ +#include +#include +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" +#include "regex_internal.h" + +enum Match_Result +{ + match = 0, + nomatch = 1 +}; + +struct Regex_String_Pair +{ + char *regex; + int string_count; + char *strings[20]; + enum Match_Result expected_results[20]; +}; + + +/** + * Random regex test. Generate a random regex as well as 'str_count' strings to + * match it against. Will match using GNUNET_REGEX implementation and compare + * the result to glibc regex result. 'rx_length' has to be smaller then + * 'max_str_len'. + * + * @param rx_length length of the regular expression. + * @param max_str_len maximum length of the random strings. + * @param str_count number of generated random strings. + * + * @return 0 on success, non 0 otherwise. + */ +int +test_random (unsigned int rx_length, unsigned int max_str_len, + unsigned int str_count) +{ + unsigned int i; + char *rand_rx; + char *matching_str; + int eval; + int eval_check; + int eval_canonical; + int eval_canonical_check; + struct REGEX_INTERNAL_Automaton *dfa; + regex_t rx; + regmatch_t matchptr[1]; + char error[200]; + int result; + char *canonical_regex = NULL; + + /* At least one string is needed for matching */ + GNUNET_assert (str_count > 0); + /* The string should be at least as long as the regex itself */ + GNUNET_assert (max_str_len >= rx_length); + + /* Generate random regex and a string that matches the regex */ + matching_str = GNUNET_malloc (rx_length + 1); + rand_rx = REGEX_TEST_generate_random_regex (rx_length, matching_str); + + /* Now match */ + result = 0; + for (i = 0; i < str_count; i++) + { + if (0 < i) + { + matching_str = REGEX_TEST_generate_random_string (max_str_len); + } + + /* Match string using DFA */ + dfa = REGEX_INTERNAL_construct_dfa (rand_rx, strlen (rand_rx), 0); + if (NULL == dfa) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Constructing DFA failed\n"); + goto error; + } + + eval = REGEX_INTERNAL_eval (dfa, matching_str); + /* save the canonical regex for later comparison */ + canonical_regex = GNUNET_strdup (REGEX_INTERNAL_get_canonical_regex (dfa)); + REGEX_INTERNAL_automaton_destroy (dfa); + + /* Match string using glibc regex */ + if (0 != regcomp (&rx, rand_rx, REG_EXTENDED)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not compile regex using regcomp: %s\n", rand_rx); + goto error; + } + + eval_check = regexec (&rx, matching_str, 1, matchptr, 0); + regfree (&rx); + + /* We only want to match the whole string, because that's what our DFA does, + * too. */ + if (eval_check == 0 && + (matchptr[0].rm_so != 0 || matchptr[0].rm_eo != strlen (matching_str))) + eval_check = 1; + + /* Match canonical regex */ + dfa = + REGEX_INTERNAL_construct_dfa (canonical_regex, strlen (canonical_regex), + 0); + if (NULL == dfa) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Constructing DFA failed\n"); + goto error; + } + + eval_canonical = REGEX_INTERNAL_eval (dfa, matching_str); + REGEX_INTERNAL_automaton_destroy (dfa); + + if (0 != regcomp (&rx, canonical_regex, REG_EXTENDED)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not compile regex using regcomp: %s\n", + canonical_regex); + goto error; + } + + eval_canonical_check = regexec (&rx, matching_str, 1, matchptr, 0); + regfree (&rx); + + /* We only want to match the whole string, because that's what our DFA does, + * too. */ + if (eval_canonical_check == 0 && + (matchptr[0].rm_so != 0 || matchptr[0].rm_eo != strlen (matching_str))) + eval_canonical_check = 1; + + /* compare results */ + if (eval_check != eval || eval_canonical != eval_canonical_check) + { + regerror (eval_check, &rx, error, sizeof error); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Unexpected result:\nregex: %s\ncanonical_regex: %s\n\ + string: %s\ngnunet regex: %i\nglibc regex: %i\n\ + canonical regex: %i\ncanonical regex glibc: %i\n\ + glibc error: %s\n\n", rand_rx, canonical_regex, matching_str, + eval, eval_check, eval_canonical, eval_canonical_check, error); + result += 1; + } + GNUNET_free (canonical_regex); + GNUNET_free (matching_str); + canonical_regex = NULL; + matching_str = NULL; + } + + GNUNET_free (rand_rx); + + return result; + +error: + GNUNET_free_non_null (matching_str); + GNUNET_free_non_null (rand_rx); + GNUNET_free_non_null (canonical_regex); + return -1; +} + +/** + * Automaton test that compares the result of matching regular expression 'rx' + * with the strings and expected results in 'rxstr' with the result of matching + * the same strings with glibc regex. + * + * @param a automaton. + * @param rx compiled glibc regex. + * @param rxstr regular expression and strings with expected results to + * match against. + * + * @return 0 on successfull, non 0 otherwise + */ +int +test_automaton (struct REGEX_INTERNAL_Automaton *a, regex_t * rx, + struct Regex_String_Pair *rxstr) +{ + int result; + int eval; + int eval_check; + char error[200]; + regmatch_t matchptr[1]; + int i; + + if (NULL == a) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Automaton was NULL\n"); + return 1; + } + + result = 0; + + for (i = 0; i < rxstr->string_count; i++) + { + eval = REGEX_INTERNAL_eval (a, rxstr->strings[i]); + eval_check = regexec (rx, rxstr->strings[i], 1, matchptr, 0); + + /* We only want to match the whole string, because that's what our DFA does, + * too. */ + if (eval_check == 0 && + (matchptr[0].rm_so != 0 || + matchptr[0].rm_eo != strlen (rxstr->strings[i]))) + eval_check = 1; + + if ((rxstr->expected_results[i] == match && (0 != eval || 0 != eval_check)) + || (rxstr->expected_results[i] == nomatch && + (0 == eval || 0 == eval_check))) + { + result = 1; + regerror (eval_check, rx, error, sizeof error); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Unexpected result:\nregex: %s\ncanonical_regex: %s\n" + "string: %s\nexpected result: %i\n" + "gnunet regex: %i\nglibc regex: %i\nglibc error: %s\n" + "rm_so: %i\nrm_eo: %i\n\n", rxstr->regex, + REGEX_INTERNAL_get_canonical_regex (a), rxstr->strings[i], + rxstr->expected_results[i], eval, eval_check, error, + matchptr[0].rm_so, matchptr[0].rm_eo); + } + } + return result; +} + +int +main (int argc, char *argv[]) +{ + GNUNET_log_setup ("test-regex", "WARNING", NULL); + + struct REGEX_INTERNAL_Automaton *a; + regex_t rx; + int i; + int check_nfa; + int check_dfa; + int check_rand; + char *check_proof; + + struct Regex_String_Pair rxstr[19] = { + {"ab?(abcd)?", 5, + {"ababcd", "abab", "aabcd", "a", "abb"}, + {match, nomatch, match, match, nomatch}}, + {"ab(c|d)+c*(a(b|c)d)+", 5, + {"abcdcdcdcdddddabd", "abcd", + "abcddddddccccccccccccccccccccccccabdacdabd", + "abccccca", "abcdcdcdccdabdabd"}, + {match, nomatch, match, nomatch, match}}, + {"ab+c*(a(bx|c)d)+", 5, + {"abcdcdcdcdddddabd", "abcd", + "abcddddddccccccccccccccccccccccccabdacdabd", + "abccccca", "abcdcdcdccdabdabd"}, + {nomatch, nomatch, nomatch, nomatch, nomatch}}, + {"a+X*y+c|p|R|Z*K*y*R+w|Y*6+n+h*k*w+V*F|W*B*e*", 1, + {"kaXycQepRZKyRwY6nhkwVFWBegNVtLPj39XhJJ6bEifRSZRYZg"}, + {nomatch}}, + {"k|a+X*y+c|Q*e|p|R|Z*K*y*R+w|Y*6+n+h*k*w+V*F|W*B*e*g|N+V|t+L|P*j*3*9+X*h*J|J*6|b|E*i*f*R+S|Z|R|Y*Z|g*", 1, + {"kaXycQepRZKyRwY6nhkwVFWBegNVtLPj39XhJJ6bEifRSZRYZg"}, + {nomatch}}, + {"F?W+m+2*6*c*s|P?U?a|B|y*i+t+A|V|6*C*7*e?Z*n*i|J?5+g?W*V?7*j?p?1|r?B?C+E+3+6*i+W*P?K?0|D+7?y*m+3?g?K?", 1, + {"osfjsodfonONONOnosndfsdnfsd"}, + {nomatch}}, + {"V|M*o?x*p*d+h+b|E*m?h?Y*E*O?W*W*P+o?Z+H*M|I*q+C*a+5?5*9|b?z|G*y*k?R|p+u|8*h?B+l*H|e|L*O|1|F?v*0?5|C+", 1, + {"VMoxpdhbEmhYEOWWPoZHMIqCa559bzGykRpu8hBlHeLO1Fv05C"}, + {nomatch}}, + {"(bla)*", 8, + {"", "bla", "blabla", "bl", "la", "b", "l", "a"}, + {match, match, match, nomatch, nomatch, nomatch, nomatch, nomatch}}, + {"ab(c|d)+c*(a(b|c)+d)+(bla)(bla)*", 8, + {"ab", "abcabdbla", "abdcccccccccccabcbccdblablabla", "bl", "la", "b", + "l", + "a"}, + {nomatch, match, match, nomatch, nomatch, nomatch, nomatch, nomatch}}, + {"a|aa*a", 6, + {"", "a", "aa", "aaa", "aaaa", "aaaaa"}, + {nomatch, match, match, match, match, match}}, + {"ab(c|d)+c*(a(b|c)+d)+(bla)+", 1, + {"abcabdblaacdbla"}, + {nomatch}}, + {"(ac|b)+", 8, + {"b", "bb", "ac", "", "acb", "bacbacac", "acacac", "abc"}, + {match, match, match, nomatch, match, match, match, nomatch}}, + {"(ab|c)+", 7, + {"", "ab", "c", "abc", "ababcc", "acc", "abac"}, + {nomatch, match, match, match, match, nomatch, nomatch}}, + {"((j|2j)K|(j|2j)AK|(j|2j)(D|e|(j|2j)A(D|e))D*K)", 1, + {"", "2j2jADK", "j2jADK"}, + {nomatch, match, match}}, + {"((j|2j)K|(j|2j)(D|e|((j|2j)j|(j|2j)2j)A(D|e))D*K|(j|2j)AK)", 2, + {"", "2j2jjADK", "j2jADK"}, + {nomatch, match, match}}, + {"ab(c|d)+c*(a(b|c)d)+", 1, + {"abacd"}, + {nomatch}}, + {"d|5kl", 1, + {"d5kl"}, + {nomatch}}, + {"a()b", 1, + {"ab"}, + {match}}, + {"GNVPN-0001-PAD(001110101001001010(0|1)*|001110101001001010000(0|1)*|001110101001001010001(0|1)*|001110101001001010010(0|1)*|001110101001001010011(0|1)*|001110101001001010100(0|1)*|001110101001001010101(0|1)*|001110101001001010110(0|1)*|001110101001001010111(0|1)*|0011101010110110(0|1)*|001110101011011000000(0|1)*|001110101011011000001(0|1)*|001110101011011000010(0|1)*|001110101011011000011(0|1)*|001110101011011000100(0|1)*|001110101011011000101(0|1)*|001110101011011000110(0|1)*|001110101011011000111(0|1)*|001110101011011001000(0|1)*|001110101011011001001(0|1)*|001110101011011001010(0|1)*|001110101011011001011(0|1)*|001110101011011001100(0|1)*|001110101011011001101(0|1)*|001110101011011001110(0|1)*|001110101011011001111(0|1)*|001110101011011010000(0|1)*|001110101011011010001(0|1)*|001110101011011010010(0|1)*|001110101011011010011(0|1)*|001110101011011010100(0|1)*|001110101011011010101(0|1)*|001110101011011010110(0|1)*|001110101011011010111(0|1)*|001110101011011011000(0|1)*|001110101011011011001(0|1)*|001110101011011011010(0|1)*|001110101011011011011(0|1)*|001110101011011011100(0|1)*|001110101011011011101(0|1)*|001110101011011011110(0|1)*|001110101011011011111(0|1)*|0011101110111101(0|1)*|001110111011110100000(0|1)*|001110111011110100001(0|1)*|001110111011110100010(0|1)*|001110111011110100011(0|1)*|001110111011110100100(0|1)*|001110111011110100101(0|1)*|001110111011110100110(0|1)*|001110111011110100111(0|1)*|001110111011110101000(0|1)*|001110111011110101001(0|1)*|001110111011110101010(0|1)*|001110111011110101011(0|1)*|001110111011110101100(0|1)*|001110111011110101101(0|1)*|001110111011110101110(0|1)*|001110111011110101111(0|1)*|001110111011110110000(0|1)*|001110111011110110001(0|1)*|001110111011110110010(0|1)*|001110111011110110011(0|1)*|001110111011110110100(0|1)*|001110111011110110101(0|1)*|001110111011110110110(0|1)*|001110111011110110111(0|1)*|001110111011110111000(0|1)*|001110111011110111001(0|1)*|001110111011110111010(0|1)*|001110111011110111011(0|1)*|001110111011110111100(0|1)*|001110111011110111101(0|1)*|001110111011110111110(0|1)*|0111010001010110(0|1)*|011101000101011000000(0|1)*|011101000101011000001(0|1)*|011101000101011000010(0|1)*|011101000101011000011(0|1)*|011101000101011000100(0|1)*|011101000101011000101(0|1)*|011101000101011000110(0|1)*|011101000101011000111(0|1)*|011101000101011001000(0|1)*|011101000101011001001(0|1)*|011101000101011001010(0|1)*|011101000101011001011(0|1)*|011101000101011001100(0|1)*|011101000101011001101(0|1)*|011101000101011001110(0|1)*|011101000101011001111(0|1)*|011101000101011010000(0|1)*|011101000101011010001(0|1)*|011101000101011010010(0|1)*|011101000101011010011(0|1)*|011101000101011010100(0|1)*|011101000101011010101(0|1)*|011101000101011010110(0|1)*|011101000101011010111(0|1)*|011101000101011011000(0|1)*|011101000101011011001(0|1)*|011101000101011011010(0|1)*|011101000101011011011(0|1)*|011101000101011011100(0|1)*|011101000101011011101(0|1)*|011101000101011011110(0|1)*|011101000101011011111(0|1)*|0111010001010111(0|1)*|011101000101011100000(0|1)*|011101000101011100001(0|1)*|011101000101011100010(0|1)*|011101000101011100011(0|1)*|011101000101011100100(0|1)*|011101000101011100101(0|1)*|011101000101011100110(0|1)*|011101000101011100111(0|1)*|011101000101011101000(0|1)*|011101000101011101001(0|1)*|011101000101011101010(0|1)*|011101000101011101011(0|1)*|011101000101011101100(0|1)*|011101000101011101101(0|1)*|011101000101011101110(0|1)*|011101000101011101111(0|1)*|011101000101011110000(0|1)*|011101000101011110001(0|1)*|011101000101011110010(0|1)*|011101000101011110011(0|1)*|011101000101011110100(0|1)*|011101000101011110101(0|1)*|011101000101011110110(0|1)*|011101000101011110111(0|1)*|011101000101011111000(0|1)*|011101000101011111001(0|1)*|011101000101011111010(0|1)*|011101000101011111011(0|1)*|011101000101011111100(0|1)*|011101000101011111101(0|1)*|011101000101011111110(0|1)*|011101000101011111111(0|1)*|0111010001011000(0|1)*|011101000101100000000(0|1)*|011101000101100000001(0|1)*|011101000101100000010(0|1)*|011101000101100000011(0|1)*|011101000101100000100(0|1)*|011101000101100000101(0|1)*|011101000101100000110(0|1)*|011101000101100000111(0|1)*|011101000101100001000(0|1)*|011101000101100001001(0|1)*|011101000101100001010(0|1)*|011101000101100001011(0|1)*|011101000101100001100(0|1)*|011101000101100001101(0|1)*|011101000101100001110(0|1)*|011101000101100001111(0|1)*|011101000101100010000(0|1)*|011101000101100010001(0|1)*|011101000101100010010(0|1)*|011101000101100010011(0|1)*|011101000101100010100(0|1)*|011101000101100010101(0|1)*|011101000101100010110(0|1)*|011101000101100010111(0|1)*|011101000101100011000(0|1)*|011101000101100011001(0|1)*|011101000101100011010(0|1)*|011101000101100011011(0|1)*|011101000101100011100(0|1)*|011101000101100011101(0|1)*|011101000101100011110(0|1)*|011101000101100011111(0|1)*|01110100010110010(0|1)*|011101000101100100000(0|1)*|011101000101100100001(0|1)*|011101000101100100010(0|1)*|011101000101100100011(0|1)*|011101000101100100100(0|1)*|011101000101100100101(0|1)*|011101000101100100110(0|1)*|011101000101100100111(0|1)*|011101000101100101000(0|1)*|011101000101100101001(0|1)*|011101000101100101010(0|1)*|011101000101100101011(0|1)*|011101000101100101100(0|1)*|011101000101100101101(0|1)*|011101000101100101110(0|1)*|011101000101100101111(0|1)*|011101000101100101111000(0|1)*|1100101010011100(0|1)*|110010101001110000000(0|1)*|110010101001110000000001(0|1)*|110010101001110000000010(0|1)*|110010101001110000000110(0|1)*|110010101001110000001(0|1)*|110010101001110000001000(0|1)*|110010101001110000001001(0|1)*|110010101001110000001010(0|1)*|110010101001110000001011(0|1)*|110010101001110000001101(0|1)*|110010101001110000001110(0|1)*|110010101001110000010(0|1)*|110010101001110000011(0|1)*|110010101001110000100(0|1)*|110010101001110000101(0|1)*|110010101001110000110(0|1)*|110010101001110000111(0|1)*|110010101001110001000(0|1)*|110010101001110001001(0|1)*|110010101001110001010(0|1)*|110010101001110001011(0|1)*|110010101001110001100(0|1)*|110010101001110001101(0|1)*|110010101001110001110(0|1)*|110010101001110001111(0|1)*|110010101001110010000(0|1)*|110010101001110010001(0|1)*|110010101001110010010(0|1)*|110010101001110010011(0|1)*|110010101001110010100(0|1)*|110010101001110010101(0|1)*|110010101001110010110(0|1)*|110010101001110010111(0|1)*|110010101001110011000(0|1)*|110010101001110011001(0|1)*|110010101001110011010(0|1)*|110010101001110011011(0|1)*|110010101001110011100(0|1)*|110010101001110011101(0|1)*|110010101001110011110(0|1)*|110010101001110011111(0|1)*|1101101010111010(0|1)*|110110101011101000000(0|1)*|110110101011101000000001(0|1)*|110110101011101000001000(0|1)*|110110101011101000001001(0|1)*|110110101011101000001010(0|1)*|110110101011101000001011(0|1)*|110110101011101000001100(0|1)*|110110101011101000001110(0|1)*|110110101011101000001111(0|1)*|110110101011101000010(0|1)*|110110101011101000010000(0|1)*|110110101011101000010001(0|1)*|110110101011101000010010(0|1)*|110110101011101000010011(0|1)*|110110101011101000011(0|1)*|110110101011101000100(0|1)*|110110101011101000101(0|1)*|110110101011101000110(0|1)*|110110101011101000111(0|1)*|110110101011101001000(0|1)*|110110101011101001001(0|1)*|110110101011101001010(0|1)*|110110101011101001011(0|1)*|110110101011101001100(0|1)*|110110101011101001101(0|1)*|110110101011101001110(0|1)*|110110101011101001111(0|1)*|110110101011101010000(0|1)*|110110101011101010001(0|1)*|110110101011101010010(0|1)*|110110101011101010011(0|1)*|110110101011101010100(0|1)*|110110101011101010101(0|1)*|110110101011101010110(0|1)*|110110101011101010111(0|1)*|110110101011101011000(0|1)*|110110101011101011001(0|1)*|110110101011101011010(0|1)*|110110101011101011011(0|1)*|110110101011101011100(0|1)*|110110101011101011101(0|1)*|110110101011101011110(0|1)*|110110101011101011111(0|1)*|1101101011010100(0|1)*|110110101101010000000(0|1)*|110110101101010000001(0|1)*|110110101101010000010(0|1)*|110110101101010000011(0|1)*|110110101101010000100(0|1)*|110110101101010000101(0|1)*|110110101101010000110(0|1)*|110110101101010000111(0|1)*|110110101101010001000(0|1)*|110110101101010001001(0|1)*|110110101101010001010(0|1)*|110110101101010001011(0|1)*|110110101101010001100(0|1)*|110110101101010001101(0|1)*|110110101101010001110(0|1)*|110110101101010001111(0|1)*|110110101101010010000(0|1)*|110110101101010010001(0|1)*|110110101101010010010(0|1)*|110110101101010010011(0|1)*|110110101101010010100(0|1)*|1101101011010100101000(0|1)*|110110101101010010101(0|1)*|110110101101010010110(0|1)*|110110101101010010111(0|1)*|110110101101010011000(0|1)*|110110101101010011010(0|1)*|110110101101010011011(0|1)*|110110101101010011100(0|1)*|110110101101010011101(0|1)*|110110101101010011110(0|1)*|110110101101010011111(0|1)*|1101111010100100(0|1)*|110111101010010000000(0|1)*|110111101010010000001(0|1)*|110111101010010000010(0|1)*|110111101010010000011(0|1)*|110111101010010000100(0|1)*|110111101010010000101(0|1)*|110111101010010000110(0|1)*|110111101010010000111(0|1)*|110111101010010001000(0|1)*|110111101010010001001(0|1)*|110111101010010001010(0|1)*|110111101010010001011(0|1)*|110111101010010001100(0|1)*|110111101010010001101(0|1)*|110111101010010001110(0|1)*|110111101010010001111(0|1)*|110111101010010010000(0|1)*|110111101010010010001(0|1)*|110111101010010010010(0|1)*|110111101010010010011(0|1)*|110111101010010010100(0|1)*|110111101010010010101(0|1)*|110111101010010010110(0|1)*|110111101010010010111(0|1)*|110111101010010011000(0|1)*|110111101010010011001(0|1)*|110111101010010011010(0|1)*|110111101010010011011(0|1)*|110111101010010011100(0|1)*|110111101010010011101(0|1)*|110111101010010011110(0|1)*|110111101010010011111(0|1)*|11011110101001010(0|1)*|110111101010010100000(0|1)*|110111101010010100001(0|1)*|110111101010010100010(0|1)*|110111101010010100011(0|1)*|110111101010010100100(0|1)*|110111101010010100101(0|1)*|110111101010010100110(0|1)*|110111101010010100111(0|1)*|110111101010010101000(0|1)*|110111101010010101001(0|1)*|110111101010010101010(0|1)*|110111101010010101011(0|1)*|110111101010010101100(0|1)*|110111101010010101101(0|1)*|110111101010010101110(0|1)*|110111101010010101111(0|1)*)", + 2, + {"GNVPN-0001-PAD1101111010100101011101010101010101", + "GNVPN-0001-PAD11001010100111000101101010101"}, + {match, match}} + }; + + check_nfa = 0; + check_dfa = 0; + check_rand = 0; + + for (i = 0; i < 19; i++) + { + if (0 != regcomp (&rx, rxstr[i].regex, REG_EXTENDED)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Could not compile regex using regcomp()\n"); + return 1; + } + + /* NFA test */ + a = REGEX_INTERNAL_construct_nfa (rxstr[i].regex, strlen (rxstr[i].regex)); + check_nfa += test_automaton (a, &rx, &rxstr[i]); + REGEX_INTERNAL_automaton_destroy (a); + + /* DFA test */ + a = REGEX_INTERNAL_construct_dfa (rxstr[i].regex, strlen (rxstr[i].regex), 0); + check_dfa += test_automaton (a, &rx, &rxstr[i]); + check_proof = GNUNET_strdup (REGEX_INTERNAL_get_canonical_regex (a)); + REGEX_INTERNAL_automaton_destroy (a); + + a = REGEX_INTERNAL_construct_dfa (check_proof, strlen (check_proof), 0); + check_dfa += test_automaton (a, &rx, &rxstr[i]); + REGEX_INTERNAL_automaton_destroy (a); + if (0 != check_dfa) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "check_proof: %s\n", check_proof); + GNUNET_free_non_null (check_proof); + + regfree (&rx); + } + + /* Random tests */ + srand (time (NULL)); + for (i = 0; i < 20; i++) + check_rand += test_random (50, 60, 10); + + return check_nfa + check_dfa + check_rand; +} diff --git a/src/regex/test_regex_graph_api.c b/src/regex/test_regex_graph_api.c new file mode 100644 index 000000000..46eacc1d7 --- /dev/null +++ b/src/regex/test_regex_graph_api.c @@ -0,0 +1,156 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_graph_api.c + * @brief test for regex_graph.c + * @author Maximilian Szengel + */ +#include +#include +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" +#include "regex_internal.h" + +#define KEEP_FILES 1 + +/** + * Check if 'filename' exists and is not empty. + * + * @param filename name of the file that should be checked + * + * @return 0 if ok, non 0 on error. + */ +static int +filecheck (const char *filename) +{ + int error = 0; + FILE *fp; + + /* Check if file was created and delete it again */ + if (NULL == (fp = fopen (filename, "r"))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not find graph %s\n", filename); + return 1; + } + + GNUNET_break (0 == fseek (fp, 0L, SEEK_END)); + if (1 > ftell (fp)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Graph writing failed, got empty file (%s)!\n", filename); + error = 2; + } + + GNUNET_assert (0 == fclose (fp)); + + if (!KEEP_FILES) + { + if (0 != unlink (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "unlink", filename); + } + return error; +} + + +int +main (int argc, char *argv[]) +{ + int error; + struct REGEX_INTERNAL_Automaton *a; + unsigned int i; + const char *filename = "test_graph.dot"; + + const char *regex[12] = { + "ab(c|d)+c*(a(b|c)+d)+(bla)+", + "(bla)*", + "b(lab)*la", + "(ab)*", + "ab(c|d)+c*(a(b|c)+d)+(bla)(bla)*", + "z(abc|def)?xyz", + "1*0(0|1)*", + "a*b*", + "a+X*y+c|p|R|Z*K*y*R+w|Y*6+n+h*k*w+V*F|W*B*e*", + "a", + "a|b", + "PADPADPADPADPADPabcdefghixxxxxxxxxxxxxjklmnop*qstoisdjfguisdfguihsdfgbdsuivggsd" + }; + + GNUNET_log_setup ("test-regex", "WARNING", NULL); + error = 0; + for (i = 0; i < 12; i++) + { + /* Check NFA graph creation */ + a = REGEX_INTERNAL_construct_nfa (regex[i], strlen (regex[i])); + REGEX_TEST_automaton_save_graph (a, filename, REGEX_TEST_GRAPH_DEFAULT); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + a = REGEX_INTERNAL_construct_nfa (regex[i], strlen (regex[i])); + REGEX_TEST_automaton_save_graph (a, filename, + REGEX_TEST_GRAPH_DEFAULT | + REGEX_TEST_GRAPH_VERBOSE); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + a = REGEX_INTERNAL_construct_nfa (regex[i], strlen (regex[i])); + REGEX_TEST_automaton_save_graph (a, filename, + REGEX_TEST_GRAPH_DEFAULT | + REGEX_TEST_GRAPH_COLORING); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + a = REGEX_INTERNAL_construct_nfa (regex[i], strlen (regex[i])); + REGEX_TEST_automaton_save_graph (a, filename, + REGEX_TEST_GRAPH_DEFAULT | + REGEX_TEST_GRAPH_VERBOSE | + REGEX_TEST_GRAPH_COLORING); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + + /* Check DFA graph creation */ + a = REGEX_INTERNAL_construct_dfa (regex[i], strlen (regex[i]), 0); + REGEX_TEST_automaton_save_graph (a, filename, REGEX_TEST_GRAPH_DEFAULT); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + a = REGEX_INTERNAL_construct_dfa (regex[i], strlen (regex[i]), 0); + REGEX_TEST_automaton_save_graph (a, filename, + REGEX_TEST_GRAPH_DEFAULT | + REGEX_TEST_GRAPH_VERBOSE); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + a = REGEX_INTERNAL_construct_dfa (regex[i], strlen (regex[i]), 0); + REGEX_TEST_automaton_save_graph (a, filename, + REGEX_TEST_GRAPH_DEFAULT | + REGEX_TEST_GRAPH_COLORING); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + + a = REGEX_INTERNAL_construct_dfa (regex[i], strlen (regex[i]), 4); + REGEX_TEST_automaton_save_graph (a, filename, REGEX_TEST_GRAPH_DEFAULT); + REGEX_INTERNAL_automaton_destroy (a); + error += filecheck (filename); + + } + + return error; +} diff --git a/src/regex/test_regex_integration.c b/src/regex/test_regex_integration.c new file mode 100644 index 000000000..99287243e --- /dev/null +++ b/src/regex/test_regex_integration.c @@ -0,0 +1,206 @@ +/* + This file is part of GNUnet. + Copyright (C) 2013, 2015 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_integration.c + * @brief base test case for regex integration with VPN; + * tests that the regexes generated by the TUN API + * for IP addresses work (for some simple cases) + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_applications.h" +#include "gnunet_util_lib.h" +#include "gnunet_tun_lib.h" +#include "gnunet_testing_lib.h" +#include "gnunet_regex_service.h" + + +/** + * How long until we really give up on a particular testcase portion? + */ +#define TOTAL_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 600) + +/** + * How long until we give up on any particular operation (and retry)? + */ +#define BASE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3) + + +static struct GNUNET_REGEX_Announcement *a4; + +static struct GNUNET_REGEX_Search *s4; + +static struct GNUNET_REGEX_Announcement *a6; + +static struct GNUNET_REGEX_Search *s6; + +static int ok = 1; + +static struct GNUNET_SCHEDULER_Task *die_task; + + +static void +end (void *cls) +{ + die_task = NULL; + GNUNET_REGEX_announce_cancel (a4); + a4 = NULL; + GNUNET_REGEX_search_cancel (s4); + s4 = NULL; + GNUNET_REGEX_announce_cancel (a6); + a6 = NULL; + GNUNET_REGEX_search_cancel (s6); + s6 = NULL; + ok = 0; +} + + +static void +end_badly () +{ + FPRINTF (stderr, "%s", "Testcase failed (timeout).\n"); + end (NULL); + ok = 1; +} + + +/** + * Search callback function, invoked for every result that was found. + * + * @param cls Closure provided in #GNUNET_REGEX_search(). + * @param id Peer providing a regex that matches the string. + * @param get_path Path of the get request. + * @param get_path_length Length of @a get_path. + * @param put_path Path of the put request. + * @param put_path_length Length of the @a put_path. + */ +static void +found_cb (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length) +{ + const char *str = cls; + static int found; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "IPv%s-exit found\n", + str); + if (0 == strcmp (str, "4")) + found |= 4; + if (0 == strcmp (str, "6")) + found |= 2; + if ((4|2) == found) + { + GNUNET_SCHEDULER_cancel (die_task); + die_task = + GNUNET_SCHEDULER_add_now (&end, NULL); + } +} + + +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Peer *peer) +{ + char rxstr4[GNUNET_TUN_IPV4_REGEXLEN]; + char rxstr6[GNUNET_TUN_IPV6_REGEXLEN]; + char *p4r; + char *p6r; + char *p4; + char *p6; + char *ss4; + char *ss6; + struct in_addr i4; + struct in6_addr i6; + + die_task = + GNUNET_SCHEDULER_add_delayed (TOTAL_TIMEOUT, + &end_badly, NULL); + GNUNET_assert (1 == + inet_pton (AF_INET, + "127.0.0.1", + &i4)); + GNUNET_assert (1 == + inet_pton (AF_INET6, + "::1:5", + &i6)); + GNUNET_TUN_ipv4toregexsearch (&i4, + 8080, + rxstr4); + GNUNET_TUN_ipv6toregexsearch (&i6, + 8686, + rxstr6); + GNUNET_asprintf (&ss4, + "%s%s", + GNUNET_APPLICATION_TYPE_EXIT_REGEX_PREFIX, + rxstr4); + GNUNET_asprintf (&ss6, + "%s%s", + GNUNET_APPLICATION_TYPE_EXIT_REGEX_PREFIX, + rxstr6); + p4r = GNUNET_TUN_ipv4policy2regex ("0.0.0.0/0:!25;"); + p6r = GNUNET_TUN_ipv6policy2regex ("::/0:!25;"); + GNUNET_asprintf (&p4, + "%s%s", + GNUNET_APPLICATION_TYPE_EXIT_REGEX_PREFIX, + p4r); + GNUNET_asprintf (&p6, + "%s%s", + GNUNET_APPLICATION_TYPE_EXIT_REGEX_PREFIX, + p6r); + GNUNET_free (p4r); + GNUNET_free (p6r); + a4 = GNUNET_REGEX_announce (cfg, + p4, + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 5), + 1); + a6 = GNUNET_REGEX_announce (cfg, + p6, + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 5), + 1); + GNUNET_free (p4); + GNUNET_free (p6); + + s4 = GNUNET_REGEX_search (cfg, + ss4, + &found_cb, "4"); + s6 = GNUNET_REGEX_search (cfg, + ss6, + &found_cb, "6"); + GNUNET_free (ss4); + GNUNET_free (ss6); +} + + +int +main (int argc, char *argv[]) +{ + if (0 != GNUNET_TESTING_peer_run ("test-regex-integration", + "test_regex_api_data.conf", + &run, NULL)) + return 1; + return ok; +} + +/* end of test_regex_integration.c */ diff --git a/src/regex/test_regex_iterate_api.c b/src/regex/test_regex_iterate_api.c new file mode 100644 index 000000000..f4cc725e0 --- /dev/null +++ b/src/regex/test_regex_iterate_api.c @@ -0,0 +1,258 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_iterate_api.c + * @brief test for regex.c + * @author Maximilian Szengel + */ +#include +#include +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_block_lib.h" +#include "regex_internal.h" + +/** + * Regex initial padding. + */ +#define INITIAL_PADDING "PADPADPADPADPADP" + +/** + * Set to GNUNET_YES to save a debug graph. + */ +#define REGEX_INTERNAL_ITERATE_SAVE_DEBUG_GRAPH GNUNET_NO + +static unsigned int transition_counter; + +struct IteratorContext +{ + int error; + int should_save_graph; + FILE *graph_filep; + unsigned int string_count; + char *const *strings; + unsigned int match_count; +}; + +struct RegexStringPair +{ + char *regex; + unsigned int string_count; + char *strings[20]; +}; + + +static void +key_iterator (void *cls, const struct GNUNET_HashCode *key, + const char *proof, + int accepting, unsigned int num_edges, + const struct REGEX_BLOCK_Edge *edges) +{ + unsigned int i; + struct IteratorContext *ctx = cls; + char *out_str; + char *state_id = GNUNET_strdup (GNUNET_h2s (key)); + + GNUNET_assert (NULL != proof); + if (GNUNET_YES == ctx->should_save_graph) + { + if (GNUNET_YES == accepting) + GNUNET_asprintf (&out_str, "\"%s\" [shape=doublecircle]\n", state_id); + else + GNUNET_asprintf (&out_str, "\"%s\" [shape=circle]\n", state_id); + fwrite (out_str, strlen (out_str), 1, ctx->graph_filep); + GNUNET_free (out_str); + + for (i = 0; i < num_edges; i++) + { + transition_counter++; + GNUNET_asprintf (&out_str, "\"%s\" -> \"%s\" [label = \"%s (%s)\"]\n", + state_id, GNUNET_h2s (&edges[i].destination), + edges[i].label, proof); + fwrite (out_str, strlen (out_str), 1, ctx->graph_filep); + + GNUNET_free (out_str); + } + } + else + { + for (i = 0; i < num_edges; i++) + transition_counter++; + } + + for (i = 0; i < ctx->string_count; i++) + { + if (0 == strcmp (proof, ctx->strings[i])) + ctx->match_count++; + } + + if (GNUNET_OK != REGEX_BLOCK_check_proof (proof, strlen (proof), key)) + { + ctx->error++; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Proof check failed: proof: %s key: %s\n", proof, state_id); + } + GNUNET_free (state_id); +} + + +int +main (int argc, char *argv[]) +{ + GNUNET_log_setup ("test-regex", "WARNING", NULL); + + int error; + struct REGEX_INTERNAL_Automaton *dfa; + unsigned int i; + unsigned int num_transitions; + char *filename = NULL; + struct IteratorContext ctx = { 0, 0, NULL, 0, NULL, 0 }; + + error = 0; + + const struct RegexStringPair rxstr[13] = { + {INITIAL_PADDING "ab(c|d)+c*(a(b|c)+d)+(bla)+", 2, + {INITIAL_PADDING "abcdcdca", INITIAL_PADDING "abcabdbl"}}, + {INITIAL_PADDING + "abcdefghixxxxxxxxxxxxxjklmnop*qstoisdjfguisdfguihsdfgbdsuivggsd", 1, + {INITIAL_PADDING "abcdefgh"}}, + {INITIAL_PADDING "VPN-4-1(0|1)*", 2, + {INITIAL_PADDING "VPN-4-10", INITIAL_PADDING "VPN-4-11"}}, + {INITIAL_PADDING "(a+X*y+c|p|R|Z*K*y*R+w|Y*6+n+h*k*w+V*F|W*B*e*)", 2, + {INITIAL_PADDING "aaaaaaaa", INITIAL_PADDING "aaXXyyyc"}}, + {INITIAL_PADDING "a*", 1, {INITIAL_PADDING "aaaaaaaa"}}, + {INITIAL_PADDING "xzxzxzxzxz", 1, {INITIAL_PADDING "xzxzxzxz"}}, + {INITIAL_PADDING "xyz*", 1, {INITIAL_PADDING "xyzzzzzz"}}, + {INITIAL_PADDING + "abcd:(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1):(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)(0|1)", + 2, {INITIAL_PADDING "abcd:000", INITIAL_PADDING "abcd:101"}}, + {INITIAL_PADDING "(x*|(0|1|2)(a|b|c|d)+)", 2, + {INITIAL_PADDING "xxxxxxxx", INITIAL_PADDING "0abcdbad"}}, + {INITIAL_PADDING "(0|1)(0|1)23456789ABC", 1, {INITIAL_PADDING "11234567"}}, + {INITIAL_PADDING "0*123456789ABC*", 3, + {INITIAL_PADDING "00123456", INITIAL_PADDING "00000000", + INITIAL_PADDING "12345678"}}, + {INITIAL_PADDING "0123456789A*BC", 1, {INITIAL_PADDING "01234567"}}, + {"GNUNETVPN000100000IPEX6-fc5a:4e1:c2ba::1", 1, {"GNUNETVPN000100000IPEX6-"}} + }; + + const char *graph_start_str = "digraph G {\nrankdir=LR\n"; + const char *graph_end_str = "\n}\n"; + + for (i = 0; i < 13; i++) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Iterating DFA for regex %s\n", + rxstr[i].regex); + + + /* Create graph */ + if (GNUNET_YES == REGEX_INTERNAL_ITERATE_SAVE_DEBUG_GRAPH) + { + GNUNET_asprintf (&filename, "iteration_graph_%u.dot", i); + ctx.graph_filep = fopen (filename, "w"); + if (NULL == ctx.graph_filep) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not open file %s for saving iteration graph.\n", + filename); + ctx.should_save_graph = GNUNET_NO; + } + else + { + ctx.should_save_graph = GNUNET_YES; + fwrite (graph_start_str, strlen (graph_start_str), 1, ctx.graph_filep); + } + GNUNET_free (filename); + } + else + { + ctx.should_save_graph = GNUNET_NO; + ctx.graph_filep = NULL; + } + + /* Iterate over DFA edges */ + transition_counter = 0; + ctx.string_count = rxstr[i].string_count; + ctx.strings = rxstr[i].strings; + ctx.match_count = 0; + dfa = + REGEX_INTERNAL_construct_dfa (rxstr[i].regex, strlen (rxstr[i].regex), 0); + REGEX_INTERNAL_iterate_all_edges (dfa, key_iterator, &ctx); + num_transitions = + REGEX_INTERNAL_get_transition_count (dfa) - dfa->start->transition_count; + + if (transition_counter < num_transitions) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Automaton has %d transitions, iterated over %d transitions\n", + num_transitions, transition_counter); + error += 1; + } + + if (ctx.match_count < ctx.string_count) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Missing initial states for regex %s\n", rxstr[i].regex); + error += (ctx.string_count - ctx.match_count); + } + else if (ctx.match_count > ctx.string_count) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Duplicate initial transitions for regex %s\n", + rxstr[i].regex); + error += (ctx.string_count - ctx.match_count); + } + + REGEX_INTERNAL_automaton_destroy (dfa); + + /* Finish graph */ + if (GNUNET_YES == ctx.should_save_graph) + { + fwrite (graph_end_str, strlen (graph_end_str), 1, ctx.graph_filep); + fclose (ctx.graph_filep); + ctx.graph_filep = NULL; + ctx.should_save_graph = GNUNET_NO; + } + } + + + for (i = 0; i < 13; i++) + { + ctx.string_count = rxstr[i].string_count; + ctx.strings = rxstr[i].strings; + ctx.match_count = 0; + + dfa = + REGEX_INTERNAL_construct_dfa (rxstr[i].regex, strlen (rxstr[i].regex), 0); + REGEX_INTERNAL_dfa_add_multi_strides (NULL, dfa, 2); + REGEX_INTERNAL_iterate_all_edges (dfa, key_iterator, &ctx); + + if (ctx.match_count < ctx.string_count) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Missing initial states for regex %s\n", rxstr[i].regex); + error += (ctx.string_count - ctx.match_count); + } + + REGEX_INTERNAL_automaton_destroy (dfa); + } + + error += ctx.error; + + return error; +} diff --git a/src/regex/test_regex_proofs.c b/src/regex/test_regex_proofs.c new file mode 100644 index 000000000..72e02fa07 --- /dev/null +++ b/src/regex/test_regex_proofs.c @@ -0,0 +1,170 @@ +/* + This file is part of GNUnet + Copyright (C) 2012 GNUnet e.V. + + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/** + * @file regex/test_regex_proofs.c + * @brief test for regex.c + * @author Maximilian Szengel + */ +#include "platform.h" +#include "regex_internal_lib.h" +#include "regex_test_lib.h" +#include "regex_internal.h" + + +/** + * Test if the given regex's canonical regex is the same as this canonical + * regex's canonical regex. Confused? Ok, then: 1. construct a dfa A from the + * given 'regex' 2. get the canonical regex of dfa A 3. construct a dfa B from + * this canonical regex 3. compare the canonical regex of dfa A with the + * canonical regex of dfa B. + * + * @param regex regular expression used for this test (see above). + * + * @return 0 on success, 1 on failure + */ +static unsigned int +test_proof (const char *regex) +{ + unsigned int error; + struct REGEX_INTERNAL_Automaton *dfa; + char *c_rx1; + const char *c_rx2; + + dfa = REGEX_INTERNAL_construct_dfa (regex, strlen (regex), 1); + GNUNET_assert (NULL != dfa); + c_rx1 = GNUNET_strdup (REGEX_INTERNAL_get_canonical_regex (dfa)); + REGEX_INTERNAL_automaton_destroy (dfa); + dfa = REGEX_INTERNAL_construct_dfa (c_rx1, strlen (c_rx1), 1); + GNUNET_assert (NULL != dfa); + c_rx2 = REGEX_INTERNAL_get_canonical_regex (dfa); + + error = (0 == strcmp (c_rx1, c_rx2)) ? 0 : 1; + + if (error > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Comparing canonical regex of\n%s\nfailed:\n%s\nvs.\n%s\n", + regex, c_rx1, c_rx2); + } + + GNUNET_free (c_rx1); + REGEX_INTERNAL_automaton_destroy (dfa); + + return error; +} + + +/** + * Use 'test_proof' function to randomly test the canonical regexes of 'count' + * random expressions of length 'rx_length'. + * + * @param count number of random regular expressions to test. + * @param rx_length length of the random regular expressions. + * + * @return 0 on succes, number of failures otherwise. + */ +static unsigned int +test_proofs_random (unsigned int count, size_t rx_length) +{ + unsigned int i; + char *rand_rx; + unsigned int failures; + + failures = 0; + + for (i = 0; i < count; i++) + { + rand_rx = REGEX_TEST_generate_random_regex (rx_length, NULL); + failures += test_proof (rand_rx); + GNUNET_free (rand_rx); + } + + return failures; +} + + +/** + * Test a number of known examples of regexes for proper canonicalization. + * + * @return 0 on success, number of failures otherwise. + */ +static unsigned int +test_proofs_static () +{ + unsigned int i; + unsigned int error; + + const char *regex[8] = { + "a|aa*a", + "a+", + "a*", + "a*a*", + "(F*C|WfPf|y+F*C)", + "y*F*C|WfPf", + "((a|b)c|(a|b)(d|(a|b)e))", + "((a|b)(c|d)|(a|b)(a|b)e)" + }; + + const char *canon_rx1; + const char *canon_rx2; + struct REGEX_INTERNAL_Automaton *dfa1; + struct REGEX_INTERNAL_Automaton *dfa2; + + error = 0; + + for (i = 0; i < 8; i += 2) + { + dfa1 = REGEX_INTERNAL_construct_dfa (regex[i], strlen (regex[i]), 1); + dfa2 = REGEX_INTERNAL_construct_dfa (regex[i + 1], strlen (regex[i + 1]), 1); + GNUNET_assert (NULL != dfa1); + GNUNET_assert (NULL != dfa2); + + canon_rx1 = REGEX_INTERNAL_get_canonical_regex (dfa1); + canon_rx2 = REGEX_INTERNAL_get_canonical_regex (dfa2); + + error += (0 == strcmp (canon_rx1, canon_rx2)) ? 0 : 1; + + if (error > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Comparing canonical regex failed:\nrx1:\t%s\ncrx1:\t%s\nrx2:\t%s\ncrx2:\t%s\n", + regex[i], canon_rx1, regex[i + 1], canon_rx2); + } + + REGEX_INTERNAL_automaton_destroy (dfa1); + REGEX_INTERNAL_automaton_destroy (dfa2); + } + + return error; +} + + +int +main (int argc, char *argv[]) +{ + GNUNET_log_setup ("test-regex", "WARNING", NULL); + + int error; + + error = 0; + + error += test_proofs_static (); + error += test_proofs_random (100, 30); + + return error; +} diff --git a/src/tun/.gitignore b/src/tun/.gitignore new file mode 100644 index 000000000..b26685596 --- /dev/null +++ b/src/tun/.gitignore @@ -0,0 +1,2 @@ +test_regex +test_tun diff --git a/src/tun/Makefile.am b/src/tun/Makefile.am new file mode 100644 index 000000000..c741f5654 --- /dev/null +++ b/src/tun/Makefile.am @@ -0,0 +1,46 @@ +# This Makefile.am is in the public domain +AM_CPPFLAGS = -I$(top_srcdir)/src/include + +if MINGW + WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +endif + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIB = -lgcov +endif + +lib_LTLIBRARIES = libgnunettun.la + +libgnunettun_la_SOURCES = \ + tun.c \ + regex.c +libgnunettun_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIB) \ + $(LTLIBINTL) +libgnunettun_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) \ + -version-info 1:0:1 + + +check_PROGRAMS = \ + test_tun \ + test_regex + +if ENABLE_TEST_RUN +AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME; +TESTS = $(check_PROGRAMS) +endif + +test_tun_SOURCES = \ + test_tun.c +test_tun_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + libgnunettun.la + + +test_regex_SOURCES = \ + test_regex.c +test_regex_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + libgnunettun.la diff --git a/src/util/regex.c b/src/tun/regex.c similarity index 100% rename from src/util/regex.c rename to src/tun/regex.c diff --git a/src/util/test_regex.c b/src/tun/test_regex.c similarity index 100% rename from src/util/test_regex.c rename to src/tun/test_regex.c diff --git a/src/util/test_tun.c b/src/tun/test_tun.c similarity index 100% rename from src/util/test_tun.c rename to src/tun/test_tun.c diff --git a/src/util/tun.c b/src/tun/tun.c similarity index 99% rename from src/util/tun.c rename to src/tun/tun.c index 7c35a76da..f85f72209 100644 --- a/src/util/tun.c +++ b/src/tun/tun.c @@ -23,7 +23,7 @@ * @author Christian Grothoff */ #include "platform.h" -#include "gnunet_util_lib.h" +#include "gnunet_tun_lib.h" /** * IP TTL we use for packets that we assemble (8 bit unsigned integer) diff --git a/src/util/Makefile.am b/src/util/Makefile.am index ec7bcb016..4296199db 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -89,8 +89,6 @@ libgnunetutil_la_SOURCES = \ crypto_rsa.c \ disk.c \ disk.h \ - dnsparser.c \ - dnsstub.c \ getopt.c \ getopt_helpers.c \ helper.c \ @@ -106,14 +104,12 @@ libgnunetutil_la_SOURCES = \ peer.c \ plugin.c \ program.c \ - regex.c \ resolver_api.c resolver.h \ scheduler.c \ service.c \ signal.c \ strings.c \ time.c \ - tun.c \ speedup.c speedup.h libgnunetutil_la_LIBADD = \ @@ -121,7 +117,7 @@ libgnunetutil_la_LIBADD = \ $(LIBGCRYPT_LIBS) \ $(LTLIBICONV) \ $(LTLIBINTL) \ - -lltdl -lidn $(Z_LIBS) -lunistring $(XLIB) + -lltdl $(Z_LIBS) -lunistring $(XLIB) libgnunetutil_la_LDFLAGS = \ $(GN_LIB_LDFLAGS) \ @@ -295,22 +291,19 @@ check_PROGRAMS = \ test_crypto_rsa \ test_disk \ test_getopt \ - test_hexcoder \ test_mq \ test_os_network \ test_peer \ test_plugin \ test_program \ - test_regex \ test_resolver_api.nc \ test_scheduler \ test_scheduler_delay \ test_service \ test_strings \ test_strings_to_data \ - test_speedup \ test_time \ - test_tun \ + test_speedup \ $(BENCHMARKS) \ test_os_start_process \ test_common_logging_runtime_loglevels @@ -326,20 +319,6 @@ test_bio_SOURCES = \ test_bio_LDADD = \ libgnunetutil.la -test_hexcoder_SOURCES = \ - test_hexcoder.c -test_hexcoder_LDADD = \ - libgnunetutil.la - -test_tun_SOURCES = \ - test_tun.c -test_tun_LDADD = \ - libgnunetutil.la - -test_regex_SOURCES = \ - test_regex.c -test_regex_LDADD = \ - libgnunetutil.la test_os_start_process_SOURCES = \ test_os_start_process.c diff --git a/src/vpn/Makefile.am b/src/vpn/Makefile.am index d1f74d35b..5c16fa349 100644 --- a/src/vpn/Makefile.am +++ b/src/vpn/Makefile.am @@ -57,6 +57,7 @@ gnunet_service_vpn_SOURCES = \ gnunet-service-vpn.c gnunet_service_vpn_LDADD = \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/tun/libgnunettun.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(top_builddir)/src/cadet/libgnunetcadet.la \ $(top_builddir)/src/regex/libgnunetregex.la \ @@ -68,6 +69,7 @@ gnunet_vpn_SOURCES = \ gnunet-vpn.c gnunet_vpn_LDADD = \ libgnunetvpn.la \ + $(top_builddir)/src/tun/libgnunettun.la \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) -- 2.25.1