From a99752d30c73cd5040c9da3c05da3bbc2dc1b67e Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 15 Jun 2019 23:46:00 +0200 Subject: [PATCH] fix gnunet-gns performance issue for many egos --- ChangeLog | 3 + src/gns/gns_tld_api.c | 88 ++++++++------------------ src/identity/Makefile.am | 1 + src/identity/gnunet-service-identity.c | 71 +++++++++++++++++++++ src/identity/identity.h | 3 +- src/include/gnunet_identity_service.h | 34 ++++++++-- src/include/gnunet_protocols.h | 9 +++ 7 files changed, 143 insertions(+), 66 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7c4c81097..405a04b90 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +Sat 15 Jun 2019 11:45:35 PM CEST + Improved gnunet-gns performance for many ego scenario. -CG + Fri 14 Jun 2019 07:17:40 PM CEST Add option to gnunet-identity to display private keys. -CG diff --git a/src/gns/gns_tld_api.c b/src/gns/gns_tld_api.c index fa4c9c057..f36b31acf 100644 --- a/src/gns/gns_tld_api.c +++ b/src/gns/gns_tld_api.c @@ -72,7 +72,7 @@ struct GNUNET_GNS_LookupWithTldRequest /** * Lookup an ego with the identity service. */ - struct GNUNET_IDENTITY_Handle *id_co; + struct GNUNET_IDENTITY_EgoSuffixLookup *id_co; /** * Name of the longest matching ego found so far. @@ -191,68 +191,37 @@ lookup_with_public_key (struct GNUNET_GNS_LookupWithTldRequest *ltr, */ static void identity_zone_cb (void *cls, - struct GNUNET_IDENTITY_Ego *ego, - void **ctx, - const char *name) + const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv, + const char *ego_name) { struct GNUNET_GNS_LookupWithTldRequest *ltr = cls; struct GNUNET_CRYPTO_EcdsaPublicKey pkey; - if (NULL == ego) + ltr->id_co = NULL; + if (NULL == priv) { - if (NULL != ltr->longest_match) - { - /* Final case: TLD matches one of our egos */ - // FIXME: eat all of the match (not just TLD!) - if (0 == strcmp (ltr->name, ltr->longest_match)) - { - /* name matches ego name perfectly, only "@" remains */ - strcpy (ltr->name, GNUNET_GNS_EMPTY_LABEL_AT); - } - else - { - GNUNET_assert (strlen (ltr->longest_match) < strlen (ltr->name)); - ltr->name[strlen (ltr->name) - strlen (ltr->longest_match) - 1] = '\0'; - } - - /* if the name is of the form 'label' (and not 'label.SUBDOMAIN'), never go to the DHT */ - GNUNET_free (ltr->longest_match); - ltr->longest_match = NULL; - if (NULL == strchr (ltr->name, (unsigned char) '.')) - ltr->options = GNUNET_GNS_LO_NO_DHT; - else - ltr->options = GNUNET_GNS_LO_LOCAL_MASTER; - - GNUNET_IDENTITY_ego_get_public_key (ltr->longest_match_ego, &pkey); - GNUNET_IDENTITY_disconnect (ltr->id_co); - ltr->id_co = NULL; - lookup_with_public_key (ltr, &pkey); - } - else - { - /* no matching ego found */ - GNUNET_IDENTITY_disconnect (ltr->id_co); - ltr->id_co = NULL; - ltr->lookup_proc (ltr->lookup_proc_cls, GNUNET_NO, 0, NULL); - GNUNET_GNS_lookup_with_tld_cancel (ltr); - } + /* no matching ego found */ + ltr->lookup_proc (ltr->lookup_proc_cls, GNUNET_NO, 0, NULL); return; } - else if (NULL != name) + /* Final case: TLD matches one of our egos */ + if (0 == strcmp (ltr->name, ego_name)) { - if ((strlen (name) <= strlen (ltr->name)) && - (0 == strcmp (name, <r->name[strlen (ltr->name) - strlen (name)])) && - ((strlen (name) == strlen (ltr->name)) || - ('.' == ltr->name[strlen (ltr->name) - strlen (name) - 1])) && - ((NULL == ltr->longest_match) || - (strlen (name) > strlen (ltr->longest_match)))) - { - /* found better match, update! */ - GNUNET_free_non_null (ltr->longest_match); - ltr->longest_match = GNUNET_strdup (name); - ltr->longest_match_ego = ego; - } + /* name matches ego name perfectly, only "@" remains */ + strcpy (ltr->name, GNUNET_GNS_EMPTY_LABEL_AT); } + else + { + GNUNET_assert (strlen (ego_name) < strlen (ltr->name)); + ltr->name[strlen (ltr->name) - strlen (ego_name) - 1] = '\0'; + } + /* if the name is of the form 'label' (and not 'label.SUBDOMAIN'), never go to the DHT */ + if (NULL == strchr (ltr->name, (unsigned char) '.')) + ltr->options = GNUNET_GNS_LO_NO_DHT; + else + ltr->options = GNUNET_GNS_LO_LOCAL_MASTER; + GNUNET_CRYPTO_ecdsa_key_get_public (priv, &pkey); + lookup_with_public_key (ltr, &pkey); } @@ -336,12 +305,11 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle, } GNUNET_free (dot_tld); } - /* FIXME: this call is still shitty slow to do the longest - suffix if we have thousands of egos. We should modify - the IDENTITY API to do the longest suffix matching - inside of the identity service and not do an O(n) IPC! */ ltr->id_co = - GNUNET_IDENTITY_connect (ltr->gns_handle->cfg, &identity_zone_cb, ltr); + GNUNET_IDENTITY_ego_lookup_by_suffix (ltr->gns_handle->cfg, + ltr->name, + &identity_zone_cb, + ltr); if (NULL == ltr->id_co) { GNUNET_free (ltr->name); @@ -365,7 +333,7 @@ GNUNET_GNS_lookup_with_tld_cancel (struct GNUNET_GNS_LookupWithTldRequest *ltr) if (NULL != ltr->id_co) { - GNUNET_IDENTITY_disconnect (ltr->id_co); + GNUNET_IDENTITY_ego_lookup_by_suffix_cancel (ltr->id_co); ltr->id_co = NULL; } if (NULL != ltr->lr) diff --git a/src/identity/Makefile.am b/src/identity/Makefile.am index 6e587dc86..476d981bb 100644 --- a/src/identity/Makefile.am +++ b/src/identity/Makefile.am @@ -43,6 +43,7 @@ libgnunet_plugin_rest_identity_la_LDFLAGS = \ libgnunetidentity_la_SOURCES = \ identity_api.c \ identity_api_lookup.c \ + identity_api_suffix_lookup.c \ identity.h libgnunetidentity_la_LIBADD = \ $(top_builddir)/src/util/libgnunetutil.la \ diff --git a/src/identity/gnunet-service-identity.c b/src/identity/gnunet-service-identity.c index a675a01f0..3e92a04cd 100644 --- a/src/identity/gnunet-service-identity.c +++ b/src/identity/gnunet-service-identity.c @@ -377,6 +377,73 @@ handle_lookup_message (void *cls, } +/** + * Handler for LOOKUP message from client, sends information + * about ONE identity to the client immediately. + * + * @param cls unused + * @param message the message received + * @return #GNUNET_SYSERR if message was ill-formed + */ +static int +check_lookup_by_suffix_message (void *cls, + const struct LookupMessage *message) +{ + GNUNET_MQ_check_zero_termination (message); + return GNUNET_OK; +} + + +/** + * Handler for LOOKUP_BY_SUFFIX message from client, sends information + * about ONE identity to the client immediately. + * + * @param cls a `struct GNUNET_SERVICE_Client *` + * @param message the message received + */ +static void +handle_lookup_by_suffix_message (void *cls, + const struct LookupMessage *message) +{ + struct GNUNET_SERVICE_Client *client = cls; + const char *name; + struct GNUNET_MQ_Envelope *env; + struct Ego *lprefix; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received LOOKUP_BY_SUFFIX message from client\n"); + name = (const char *) &message[1]; + lprefix = NULL; + for (struct Ego *ego = ego_head; NULL != ego; ego = ego->next) + { + + if ((strlen (ego->identifier) <= strlen (name)) && + (0 == strcmp (ego->identifier, + &name[strlen (name) - strlen (ego->identifier)])) && + ((strlen (name) == strlen (ego->identifier)) || + ('.' == name[strlen (name) - + strlen (ego->identifier) - 1])) && + ((NULL == lprefix) || + (strlen (ego->identifier) > strlen (lprefix->identifier)))) + { + /* found better match, update! */ + lprefix = ego; + } + } + if (NULL != lprefix) + { + env = create_update_message (lprefix); + GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env); + GNUNET_SERVICE_client_continue (client); + return; + } + send_result_code (client, + 0, + "ego not found"); + GNUNET_SERVICE_client_continue (client); +} + + /** * Checks a #GNUNET_MESSAGE_TYPE_IDENTITY_GET_DEFAULT message * @@ -1118,6 +1185,10 @@ GNUNET_SERVICE_MAIN GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP, struct LookupMessage, NULL), + GNUNET_MQ_hd_var_size (lookup_by_suffix_message, + GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX, + struct LookupMessage, + NULL), GNUNET_MQ_hd_var_size (get_default_message, GNUNET_MESSAGE_TYPE_IDENTITY_GET_DEFAULT, struct GetDefaultMessage, diff --git a/src/identity/identity.h b/src/identity/identity.h index 96550bdf2..6ef16e39d 100644 --- a/src/identity/identity.h +++ b/src/identity/identity.h @@ -62,7 +62,8 @@ struct ResultCodeMessage struct LookupMessage { /** - * Type: #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP + * Type: #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP or + * #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX */ struct GNUNET_MessageHeader header; diff --git a/src/include/gnunet_identity_service.h b/src/include/gnunet_identity_service.h index b2472da43..52ed1e908 100644 --- a/src/include/gnunet_identity_service.h +++ b/src/include/gnunet_identity_service.h @@ -328,6 +328,24 @@ GNUNET_IDENTITY_ego_lookup (const struct GNUNET_CONFIGURATION_Handle *cfg, void GNUNET_IDENTITY_ego_lookup_cancel (struct GNUNET_IDENTITY_EgoLookup *el); +/** + * Function called with the result. + * + * @param cls closure + * @param ego NULL on error / ego not found + * @param ego_name NULL on error, name of the ego otherwise + */ +typedef void (*GNUNET_IDENTITY_EgoSuffixCallback) ( + void *cls, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv, + const char *ego_name); + + +/** + * Handle for suffix lookup. + */ +struct GNUNET_IDENTITY_EgoSuffixLookup; + /** * Obtain the ego with the maximum suffix match between the @@ -335,20 +353,26 @@ GNUNET_IDENTITY_ego_lookup_cancel (struct GNUNET_IDENTITY_EgoLookup *el); * a @a suffix "a.b.c" and egos with names "d.a.b.c", "b.c" and "c", * we return the ego for "b.c". * - * @param id identity service to query + * @param cfg configuration to use * @param suffix for which domain name suffix is an identity wanted * @param cb function to call with the result (will only be called once) * @param cb_cls closure for @a cb * @return handle to abort the operation */ -struct GNUNET_IDENTITY_EgoLookup * -GNUNET_IDENTITY_ego_lookup_by_suffix (struct GNUNET_IDENTITY_Handle *id, +struct GNUNET_IDENTITY_EgoSuffixLookup * +GNUNET_IDENTITY_ego_lookup_by_suffix (const struct GNUNET_CONFIGURATION_Handle *cfg, const char *suffix, - GNUNET_IDENTITY_EgoCallback cb, + GNUNET_IDENTITY_EgoSuffixCallback cb, void *cb_cls); - +/** + * Abort ego suffix lookup attempt. + * + * @param el handle for lookup to abort + */ +void +GNUNET_IDENTITY_ego_lookup_by_suffix_cancel (struct GNUNET_IDENTITY_EgoSuffixLookup *el); #if 0 /* keep Emacsens' auto-indent happy */ { diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h index d93e12bfb..45bfa4f1b 100644 --- a/src/include/gnunet_protocols.h +++ b/src/include/gnunet_protocols.h @@ -1948,6 +1948,15 @@ extern "C" { */ #define GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP 632 +/** + * First message send from identity client to service to lookup a + * single ego matching the given suffix (longest match). The service + * will respond with a #GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE message if + * the ego exists, or a #GNUNET_MESSAGE_TYPE_IDENTITY_RESULT_CODE if + * not. + */ +#define GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX 633 + /******************************************************************************* * REVOCATION message types -- 2.25.1