From: Christian Grothoff <christian@grothoff.org>
Date: Tue, 11 Dec 2018 18:27:34 +0000 (+0100)
Subject: cache NICK results in namestore to avoid unnecessary load on the DB; improve sqlite... 
X-Git-Tag: v0.11.0~168
X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=c1dd4d02762d51d1ac4078407cda2796aff8faf4;p=oweals%2Fgnunet.git

cache NICK results in namestore to avoid unnecessary load on the DB; improve sqlite DB style, use WAL for sqlite DB
---

diff --git a/src/arm/gnunet-arm.c b/src/arm/gnunet-arm.c
index 28d1f7592..fa0374bc5 100644
--- a/src/arm/gnunet-arm.c
+++ b/src/arm/gnunet-arm.c
@@ -74,7 +74,7 @@ static char *term;
 /**
  * Set to the name of the config file used.
  */
-static const char *config_file;
+static char *config_file;
 
 /**
  * Set to the directory where runtime files are stored.
@@ -172,6 +172,7 @@ delete_files ()
 static void
 shutdown_task (void *cls)
 {
+  (void) cls;
   if (NULL != al_task)
   {
     GNUNET_SCHEDULER_cancel (al_task);
@@ -284,6 +285,7 @@ conn_status (void *cls,
 {
   static int once;
 
+  (void) cls;
   if ( (GNUNET_SYSERR == connected) &&
        (0 == once) )
   {
@@ -312,6 +314,7 @@ start_callback (void *cls,
 		enum GNUNET_ARM_RequestStatus rs,
 		enum GNUNET_ARM_Result result)
 {
+  (void) cls;
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
@@ -355,6 +358,7 @@ stop_callback (void *cls,
 {
   char *msg;
 
+  (void) cls;
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
@@ -405,6 +409,7 @@ init_callback (void *cls,
 	       enum GNUNET_ARM_RequestStatus rs,
 	       enum GNUNET_ARM_Result result)
 {
+  (void) cls;
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
@@ -452,6 +457,7 @@ term_callback (void *cls,
 {
   char *msg;
 
+  (void) cls;
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
@@ -501,6 +507,7 @@ list_callback (void *cls,
 {
   unsigned int i;
 
+  (void) cls;
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
@@ -538,6 +545,7 @@ list_callback (void *cls,
 static void
 action_loop (void *cls)
 {
+  (void) cls;
   al_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 	      "Running requested actions\n");
@@ -637,6 +645,7 @@ srv_status (void *cls,
 {
   const char *msg;
 
+  (void) cls;
   switch (status)
   {
   case GNUNET_ARM_SERVICE_MONITORING_STARTED:
@@ -679,6 +688,7 @@ srv_status (void *cls,
 static void
 timeout_task_cb (void *cls)
 {
+  (void) cls;
   timeout_task = NULL;
   ret = 2;
   GNUNET_SCHEDULER_shutdown ();
@@ -699,8 +709,9 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  char *armconfig;
-
+  (void) cls;
+  (void) args;
+  (void) cfgfile;
   cfg = GNUNET_CONFIGURATION_dup (c);
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index f67831708..b355d37ba 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -168,6 +168,18 @@ create_indices (sqlite3 * dbh)
 static int
 database_setup (struct Plugin *plugin)
 {
+  struct GNUNET_SQ_ExecuteStatement es[] = {
+    GNUNET_SQ_make_try_execute ("PRAGMA temp_store=MEMORY"),
+    GNUNET_SQ_make_try_execute ("PRAGMA synchronous=NORMAL"),
+    GNUNET_SQ_make_try_execute ("PRAGMA legacy_file_format=OFF"),
+    GNUNET_SQ_make_try_execute ("PRAGMA auto_vacuum=INCREMENTAL"),
+    GNUNET_SQ_make_try_execute ("PRAGMA encoding=\"UTF-8\""),
+    GNUNET_SQ_make_try_execute ("PRAGMA locking_mode=EXCLUSIVE"),
+    GNUNET_SQ_make_try_execute ("PRAGMA page_size=4092"),
+    GNUNET_SQ_make_try_execute ("PRAGMA journal_mode=WAL"),
+    GNUNET_SQ_EXECUTE_STATEMENT_END
+  };
+  
   sqlite3_stmt *stmt;
   char *afsdir;
 #if ENULL_DEFINED
@@ -208,47 +220,16 @@ database_setup (struct Plugin *plugin)
 	 sqlite3_errmsg (plugin->dbh));
     return GNUNET_SYSERR;
   }
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA temp_store=MEMORY",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA synchronous=NORMAL",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA legacy_file_format=OFF",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA auto_vacuum=INCREMENTAL",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA encoding=\"UTF-8\"",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA locking_mode=EXCLUSIVE",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA page_size=4092",
-                       NULL, NULL,
-                       ENULL));
-  CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh,
-                       "PRAGMA journal_mode=WAL",
-                       NULL, NULL,
-                       ENULL));
-
+  if (GNUNET_OK !=
+      GNUNET_SQ_exec_statements (plugin->dbh,
+                                 es))
+  {
+    GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Failed to setup database at `%s'\n"),
+         plugin->fn);
+    return GNUNET_SYSERR;
+  }
   CHECK (SQLITE_OK ==
          sqlite3_busy_timeout (plugin->dbh,
                                BUSY_TIMEOUT_MS));
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index d9fa5b3fc..d9e9a9e82 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -46,6 +46,10 @@
  */
 #define MONITOR_STALL_WARN_DELAY GNUNET_TIME_UNIT_MINUTES
 
+/**
+ * Size of the cache used by #get_nick_record()
+ */
+#define NC_SIZE 16
 
 /**
  * A namestore client
@@ -291,6 +295,33 @@ struct StoreActivity
 };
 
 
+/**
+ * Entry in list of cached nick resolutions.
+ */ 
+struct NickCache
+{
+  /**
+   * Zone the cache entry is for.
+   */ 
+  struct GNUNET_CRYPTO_EcdsaPrivateKey zone;
+
+  /**
+   * Cached record data.
+   */
+  struct GNUNET_GNSRECORD_Data *rd;
+
+  /**
+   * Timestamp when this cache entry was used last.
+   */
+  struct GNUNET_TIME_Absolute last_used;
+};
+
+
+/**
+ * We cache nick records to reduce DB load. 
+ */
+static struct NickCache nick_cache[NC_SIZE];
+
 /**
  * Public key of all zeros.
  */
@@ -480,6 +511,48 @@ lookup_nick_it (void *cls,
 }
 
 
+/**
+ * Add entry to the cache for @a zone and @a nick
+ *
+ * @param zone zone key to cache under
+ * @param nick nick entry to cache
+ */
+static void
+cache_nick (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
+	    const struct GNUNET_GNSRECORD_Data *nick)
+{
+  struct NickCache *oldest;
+
+  oldest = NULL;
+  for (unsigned int i=0;i<NC_SIZE;i++)
+  {
+    struct NickCache *pos = &nick_cache[i];
+
+    if ( (NULL == oldest) ||
+	 (oldest->last_used.abs_value_us >
+	  pos->last_used.abs_value_us) )
+      oldest = pos;
+    if (0 == memcmp (zone,
+		     &pos->zone,
+		     sizeof (*zone)))
+    {
+      oldest = pos;
+      break;
+    }
+  }
+  GNUNET_free_non_null (oldest->rd);
+  oldest->zone = *zone;
+  oldest->rd = GNUNET_malloc (sizeof (*nick) +
+			      nick->data_size);
+  *oldest->rd = *nick;
+  oldest->rd->data = &oldest->rd[1];
+  memcpy (&oldest->rd[1],
+	  nick->data,
+	  nick->data_size);
+  oldest->last_used = GNUNET_TIME_absolute_get ();
+}
+
+
 /**
  * Return the NICK record for the zone (if it exists).
  *
@@ -493,6 +566,27 @@ get_nick_record (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone)
   struct GNUNET_GNSRECORD_Data *nick;
   int res;
 
+  /* check cache first */
+  for (unsigned int i=0;i<NC_SIZE;i++)
+  {
+    struct NickCache *pos = &nick_cache[i];
+    if ( (NULL != pos->rd) &&
+	 (0 == memcmp (zone,
+		       &pos->zone,
+		       sizeof (*zone))) )
+    {
+      nick = GNUNET_malloc (sizeof (*nick) +
+			    pos->rd->data_size);
+      *nick = *pos->rd;
+      nick->data = &nick[1];
+      memcpy (&nick[1],
+	      pos->rd->data,
+	      pos->rd->data_size);
+      pos->last_used = GNUNET_TIME_absolute_get ();
+      return nick;
+    }
+  }
+  
   nick = NULL;
   res = GSN_database->lookup_records (GSN_database->cls,
 				      zone,
@@ -508,6 +602,10 @@ get_nick_record (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone)
                 GNUNET_GNSRECORD_z2s (&pub));
     return NULL;
   }
+
+  /* update cache */
+  cache_nick (zone,
+	      nick);
   return nick;
 }
 
@@ -663,7 +761,7 @@ send_lookup_response (struct NamestoreClient *nc,
     GNUNET_SERVICE_client_drop (nc->client);
     return;
   }
-  if (rd_ser_len >= UINT16_MAX - name_len - sizeof (*zir_msg))
+  if (((size_t) rd_ser_len) >= UINT16_MAX - name_len - sizeof (*zir_msg))
   {
     GNUNET_break (0);
     GNUNET_SERVICE_client_drop (nc->client);
@@ -1445,6 +1543,12 @@ handle_record_store (void *cls,
                            conv_name)) ||
              (GNUNET_GNSRECORD_TYPE_NICK != rd[i].record_type) )
           rd_clean_off++;
+	
+	if ( (0 == strcmp (GNUNET_GNS_EMPTY_LABEL_AT,
+                           conv_name)) &&
+	     (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) )
+	  cache_nick (&rp_msg->private_key,
+		      &rd[i]);
       }
       res = GSN_database->store_records (GSN_database->cls,
 					 &rp_msg->private_key,
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index 23af3960d..e68a47a7b 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -122,6 +122,7 @@ database_setup (struct Plugin *plugin)
     GNUNET_SQ_make_try_execute ("PRAGMA auto_vacuum=INCREMENTAL"),
     GNUNET_SQ_make_try_execute ("PRAGMA encoding=\"UTF-8\""),
     GNUNET_SQ_make_try_execute ("PRAGMA locking_mode=EXCLUSIVE"),
+    GNUNET_SQ_make_try_execute ("PRAGMA journal_mode=WAL"),
     GNUNET_SQ_make_try_execute ("PRAGMA page_size=4092"),
     GNUNET_SQ_make_execute ("CREATE TABLE IF NOT EXISTS ns098records ("
                             " uid INTEGER PRIMARY KEY,"