fixing calculation of Bloom filter size that was too large by 1024x because it was...
[oweals/gnunet.git] / src / datastore / test_plugin_datastore.c
index d38e908acee232dfdff2c0648d87ad102bc7d2a5..3504945aec29c79b75d15051792291ae620dfd25 100644 (file)
@@ -28,7 +28,7 @@
 #include "gnunet_protocols.h"
 #include "gnunet_datastore_plugin.h"
 
-#define VERBOSE GNUNET_NO
+#define VERBOSE GNUNET_EXTRA_LOGGING
 
 /**
  * Number of put operations to perform.
@@ -46,25 +46,26 @@ static const char *plugin_name;
 static int ok;
 
 enum RunPhase
-  {
-    RP_ERROR = 0,
-    RP_PUT,
-    RP_GET,
-    RP_UPDATE,
-    RP_ITER_ZERO,
-    RP_REPL_GET,
-    RP_EXPI_GET,
-    RP_DROP
-  };
+{
+  RP_ERROR = 0,
+  RP_PUT,
+  RP_GET,
+  RP_UPDATE,
+  RP_ITER_ZERO,
+  RP_REPL_GET,
+  RP_EXPI_GET,
+  RP_DROP
+};
 
 
 struct CpsRunContext
 {
   const struct GNUNET_CONFIGURATION_Handle *cfg;
-  struct GNUNET_DATASTORE_PluginFunctions * api;
+  struct GNUNET_DATASTORE_PluginFunctions *api;
   enum RunPhase phase;
   unsigned int cnt;
   unsigned int i;
+  uint64_t offset;
 };
 
 
@@ -73,31 +74,27 @@ struct CpsRunContext
  * change in their disk utilization.
  *
  * @param cls closure (NULL)
- * @param delta change in disk utilization, 
+ * @param delta change in disk utilization,
  *        0 for "reset to empty"
  */
 static void
-disk_utilization_change_cb (void *cls,
-                           int delta)
+disk_utilization_change_cb (void *cls, int delta)
 {
   /* do nothing */
 }
 
 
 static void
-gen_key (int i,
-        GNUNET_HashCode *key)
+gen_key (int i, GNUNET_HashCode * key)
 {
   memset (key, 0, sizeof (GNUNET_HashCode));
   key->bits[0] = (unsigned int) i;
   GNUNET_CRYPTO_hash (key, sizeof (GNUNET_HashCode), key);
 }
 
-            
+
 static void
-put_value (struct GNUNET_DATASTORE_PluginFunctions * api,
-          int i, 
-          int k)
+put_value (struct GNUNET_DATASTORE_PluginFunctions *api, int i, int k)
 {
   char value[65536];
   size_t size;
@@ -108,7 +105,7 @@ put_value (struct GNUNET_DATASTORE_PluginFunctions * api,
   /* most content is 32k */
   size = 32 * 1024;
 
-  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0)  /* but some of it is less! */
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0)   /* but some of it is less! */
     size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
   size = size - (size & 7);     /* always multiple of 8 */
 
@@ -120,24 +117,24 @@ put_value (struct GNUNET_DATASTORE_PluginFunctions * api,
   value[0] = k;
   msg = NULL;
   prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
-  if (GNUNET_OK != api->put (api->cls,
-                            &key, 
-                            size,
-                            value,
-                            i + 1 /* type */,
-                            prio,
-                            i /* anonymity */,
-                            0 /* replication */,
-                            GNUNET_TIME_relative_to_absolute 
-                            (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                                            60 * 60 * 60 * 1000 +
-                                                            GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
-                            &msg))
-    {
-      fprintf (stderr, "ERROR: `%s'\n", msg);
-      GNUNET_free_non_null (msg);
-      return;
-    }
+#if VERBOSE
+  fprintf (stderr, "putting type %u, anon %u under key %s\n", i + 1, i,
+           GNUNET_h2s (&key));
+#endif
+  if (GNUNET_OK != api->put (api->cls, &key, size, value, i + 1 /* type */ ,
+                             prio, i /* anonymity */ ,
+                             0 /* replication */ ,
+                             GNUNET_TIME_relative_to_absolute
+                             (GNUNET_TIME_relative_multiply
+                              (GNUNET_TIME_UNIT_MILLISECONDS,
+                               60 * 60 * 60 * 1000 +
+                               GNUNET_CRYPTO_random_u32
+                               (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), &msg))
+  {
+    fprintf (stderr, "ERROR: `%s'\n", msg);
+    GNUNET_free_non_null (msg);
+    return;
+  }
   stored_bytes += size;
   stored_ops++;
   stored_entries++;
@@ -145,72 +142,30 @@ put_value (struct GNUNET_DATASTORE_PluginFunctions * api,
 
 
 static void
-test (void *cls,
-      const struct GNUNET_SCHEDULER_TaskContext *tc);
-
-
-static int
-iterate_one_shot (void *cls,
-                 void *next_cls,
-                 const GNUNET_HashCode * key,
-                 uint32_t size,
-                 const void *data,
-                 enum GNUNET_BLOCK_Type type,
-                 uint32_t priority,
-                 uint32_t anonymity,
-                 struct GNUNET_TIME_Absolute
-                 expiration, 
-                 uint64_t uid)
-{
-  struct CpsRunContext *crc = cls;
-  
-  GNUNET_assert (NULL == next_cls);
-  GNUNET_assert (key != NULL);
-  crc->phase++;
-#if VERBOSE
-  fprintf (stderr,
-          "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
-          type, priority, size,
-          (unsigned long long) expiration.abs_value);
-#endif    
-  GNUNET_SCHEDULER_add_now (&test, crc);
-  return GNUNET_OK;
-}
+test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
 
 
 static uint64_t guid;
 
+
 static int
-iterate_with_next (void *cls,
-                  void *next_cls,
-                  const GNUNET_HashCode * key,
-                  uint32_t size,
-                  const void *data,
-                  enum GNUNET_BLOCK_Type type,
-                  uint32_t priority,
-                  uint32_t anonymity,
-                  struct GNUNET_TIME_Absolute
-                  expiration, 
-                  uint64_t uid)
+iterate_one_shot (void *cls, const GNUNET_HashCode * key, uint32_t size,
+                  const void *data, enum GNUNET_BLOCK_Type type,
+                  uint32_t priority, uint32_t anonymity,
+                  struct GNUNET_TIME_Absolute expiration, uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
-  
-  if (key == NULL)
-    {
-      crc->phase++;
-      GNUNET_SCHEDULER_add_now (&test, crc);
-      return GNUNET_OK;
-    }
+
+  GNUNET_assert (key != NULL);
   guid = uid;
+  crc->phase++;
 #if VERBOSE
   fprintf (stderr,
-          "Found result type=%u, priority=%u, size=%u, expire=%llu\n",
-          type, priority, size,
-          (unsigned long long) expiration.abs_value);
+           "Found result type=%u, priority=%u, size=%u, expire=%llu, key %s\n",
+           type, priority, size, (unsigned long long) expiration.abs_value,
+           GNUNET_h2s (key));
 #endif
-  crc->cnt++;
-  crc->api->next_request (next_cls,
-                         GNUNET_NO);
+  GNUNET_SCHEDULER_add_now (&test, crc);
   return GNUNET_OK;
 }
 
@@ -223,22 +178,21 @@ iterate_with_next (void *cls,
  * @param cfg configuration to use
  */
 static void
-unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api,
-              const struct GNUNET_CONFIGURATION_Handle *cfg)
+unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   char *name;
   char *libname;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_string (cfg,
-                                             "DATASTORE", "DATABASE", &name))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("No `%s' specified for `%s' in configuration!\n"),
-                 "DATABASE",
-                 "DATASTORE");
-      return;
-    }
+      GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
+                                             &name))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
+                "DATASTORE");
+    return;
+  }
   GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
   GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
   GNUNET_free (libname);
@@ -252,8 +206,7 @@ unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api,
  * the transport and core.
  */
 static void
-cleaning_task (void *cls,
-              const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct CpsRunContext *crc = cls;
 
@@ -263,9 +216,8 @@ cleaning_task (void *cls,
 
 
 static void
-test (void *cls,
-      const struct GNUNET_SCHEDULER_TaskContext *tc)
-{  
+test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
   struct CpsRunContext *crc = cls;
   int j;
   unsigned long long os;
@@ -273,82 +225,74 @@ test (void *cls,
   GNUNET_HashCode key;
 
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Test aborted.\n");
+    crc->phase = RP_ERROR;
+  }
+#if VERBOSE
+  fprintf (stderr, "In phase %d, iteration %u\n", crc->phase, crc->cnt);
+#endif
+  switch (crc->phase)
+  {
+  case RP_ERROR:
+    ok = 1;
+    GNUNET_break (0);
+    crc->api->drop (crc->api->cls);
+    GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
+    break;
+  case RP_PUT:
+    os = 0;
+    for (j = 0; j < PUT_10; j++)
     {
-      crc->phase = RP_ERROR;
-      ok = 1;
+      put_value (crc->api, j, crc->i);
+      cs = crc->api->estimate_size (crc->api->cls);
+      GNUNET_assert (os <= cs);
+      os = cs;
     }
-  switch (crc->phase)
+    crc->phase++;
+    GNUNET_SCHEDULER_add_now (&test, crc);
+    break;
+  case RP_GET:
+    if (crc->cnt == 1)
     {
-    case RP_ERROR:
-      GNUNET_break (0); 
-      crc->api->drop (crc->api->cls);
-      GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
-      break;
-    case RP_PUT:      
-      os = 0;
-      for (j=0;j<PUT_10;j++)
-       {
-         put_value (crc->api, j, crc->i);
-         cs = crc->api->get_size (crc->api->cls);
-         GNUNET_assert (os < cs);
-         os = cs;
-       }
+      crc->cnt = 0;
       crc->phase++;
       GNUNET_SCHEDULER_add_now (&test, crc);
       break;
-    case RP_GET:
-      if (crc->cnt == 1)
-       {
-         crc->cnt = 0;
-         crc->phase++;
-         GNUNET_SCHEDULER_add_now (&test, crc);
-         break;
-       }
-      gen_key (5, &key);
-      crc->api->get (crc->api->cls,
-                    &key, NULL, 
-                    GNUNET_BLOCK_TYPE_ANY,
-                    &iterate_with_next,
-                    crc);
-      break;
-    case RP_UPDATE:
-      GNUNET_assert (GNUNET_OK ==
-                    crc->api->update (crc->api->cls,
-                                      guid, 1, 
-                                      GNUNET_TIME_UNIT_ZERO_ABS,
-                                      NULL));
+    }
+    gen_key (5, &key);
+    crc->api->get_key (crc->api->cls, crc->offset++, &key, NULL,
+                       GNUNET_BLOCK_TYPE_ANY, &iterate_one_shot, crc);
+    break;
+  case RP_UPDATE:
+    GNUNET_assert (GNUNET_OK ==
+                   crc->api->update (crc->api->cls, guid, 1,
+                                     GNUNET_TIME_UNIT_ZERO_ABS, NULL));
+    crc->phase++;
+    GNUNET_SCHEDULER_add_now (&test, crc);
+    break;
+
+  case RP_ITER_ZERO:
+    if (crc->cnt == 1)
+    {
+      crc->cnt = 0;
       crc->phase++;
       GNUNET_SCHEDULER_add_now (&test, crc);
       break;
-
-    case RP_ITER_ZERO:
-      if (crc->cnt == 1)
-       {
-         crc->cnt = 0;
-         crc->phase++;
-         GNUNET_SCHEDULER_add_now (&test, crc);
-         break;
-       }
-      crc->api->iter_zero_anonymity (crc->api->cls, 
-                                    1, 
-                                    &iterate_with_next,
-                                    crc);
-      break;
-    case RP_REPL_GET:
-      crc->api->replication_get (crc->api->cls, 
-                                &iterate_one_shot,
-                                crc);
-      break;
-    case RP_EXPI_GET:
-      crc->api->expiration_get (crc->api->cls, 
-                               &iterate_one_shot,
-                               crc);
-      break;
-    case RP_DROP:
-      crc->api->drop (crc->api->cls);
-      GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
-      break;
     }
+    crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc);
+    break;
+  case RP_REPL_GET:
+    crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc);
+    break;
+  case RP_EXPI_GET:
+    crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc);
+    break;
+  case RP_DROP:
+    crc->api->drop (crc->api->cls);
+    GNUNET_SCHEDULER_add_now (&cleaning_task, crc);
+    break;
+  }
 }
 
 
@@ -359,33 +303,30 @@ static struct GNUNET_DATASTORE_PluginFunctions *
 load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   static struct GNUNET_DATASTORE_PluginEnvironment env;
-  struct GNUNET_DATASTORE_PluginFunctions * ret; 
+  struct GNUNET_DATASTORE_PluginFunctions *ret;
   char *name;
   char *libname;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_string (cfg,
-                                             "DATASTORE", "DATABASE", &name))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                 _("No `%s' specified for `%s' in configuration!\n"),
-                 "DATABASE",
-                 "DATASTORE");
-      return NULL;
-    }
+      GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
+                                             &name))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
+                "DATASTORE");
+    return NULL;
+  }
   env.cfg = cfg;
   env.duc = &disk_utilization_change_cb;
   env.cls = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              _("Loading `%s' datastore plugin\n"), name);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"),
+              name);
   GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
   if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
-    {
-      fprintf (stderr,
-              "Failed to load plugin `%s'!\n",
-              name);
-      return NULL;
-    }
+  {
+    fprintf (stderr, "Failed to load plugin `%s'!\n", name);
+    return NULL;
+  }
   GNUNET_free (libname);
   GNUNET_free (name);
   return ret;
@@ -393,9 +334,7 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
 
 static void
-run (void *cls,
-     char *const *args,
-     const char *cfgfile,
+run (void *cls, char *const *args, const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
   struct GNUNET_DATASTORE_PluginFunctions *api;
@@ -403,12 +342,12 @@ run (void *cls,
 
   api = load_plugin (c);
   if (api == NULL)
-    {
-      fprintf (stderr, 
-              "Could not initialize plugin, assuming database not configured. Test not run!\n");
-      return;
-    }
-  crc = GNUNET_malloc(sizeof(struct CpsRunContext));
+  {
+    fprintf (stderr,
+             "Could not initialize plugin, assuming database not configured. Test not run!\n");
+    return;
+  }
+  crc = GNUNET_malloc (sizeof (struct CpsRunContext));
   crc->api = api;
   crc->cfg = c;
   crc->phase = RP_PUT;
@@ -420,7 +359,8 @@ static int
 check ()
 {
   char cfg_name[128];
-  char *const argv[] = { 
+
+  char *const argv[] = {
     "test-plugin-datastore",
     "-c",
     cfg_name,
@@ -433,13 +373,10 @@ check ()
     GNUNET_GETOPT_OPTION_END
   };
 
-  GNUNET_snprintf (cfg_name,
-                  sizeof (cfg_name),
-                  "test_plugin_datastore_data_%s.conf",
-                  plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
-                      argv, "test-plugin-datastore", "nohelp",
-                      options, &run, NULL);
+  GNUNET_snprintf (cfg_name, sizeof (cfg_name),
+                   "test_plugin_datastore_data_%s.conf", plugin_name);
+  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv,
+                      "test-plugin-datastore", "nohelp", options, &run, NULL);
   if (ok != 0)
     fprintf (stderr, "Missed some testcases: %u\n", ok);
   return ok;
@@ -453,19 +390,18 @@ main (int argc, char *argv[])
   char *pos;
   char dir_name[128];
 
+  sleep (1);
   /* determine name of plugin to use */
   plugin_name = argv[0];
-  while (NULL != (pos = strstr(plugin_name, "_")))
-    plugin_name = pos+1;
-  if (NULL != (pos = strstr(plugin_name, ".")))
+  while (NULL != (pos = strstr (plugin_name, "_")))
+    plugin_name = pos + 1;
+  if (NULL != (pos = strstr (plugin_name, ".")))
     pos[0] = 0;
   else
     pos = (char *) plugin_name;
 
-  GNUNET_snprintf (dir_name,
-                  sizeof (dir_name),
-                  "/tmp/test-gnunet-datastore-%s",
-                  plugin_name);
+  GNUNET_snprintf (dir_name, sizeof (dir_name),
+                   "/tmp/test-gnunet-datastore-plugin-%s", plugin_name);
   GNUNET_DISK_directory_remove (dir_name);
   GNUNET_log_setup ("test-plugin-datastore",
 #if VERBOSE
@@ -483,5 +419,3 @@ main (int argc, char *argv[])
 }
 
 /* end of test_plugin_datastore.c */
-
-