indentation

[oweals/gnunet.git] / src / datastore / perf_datastore_api.c

diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
index 95ee18dad40fac030ea9e57df4dc57cdf1585625..cf9911329eb1d0607317aa45c8163b99e72cec59 100644 (file)
--- a/src/datastore/perf_datastore_api.c
+++ b/src/datastore/perf_datastore_api.c
@@ -1,10 +1,10 @@
 /*
      This file is part of GNUnet.
 /*
      This file is part of GNUnet.

-     (C) 2004, 2005, 2006, 2007 Christian Grothoff (and other contributing authors)
+     (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors)

 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published

-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your

      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but


@@ -22,78 +22,55 @@
  * @brief performance measurement for the datastore implementation
  * @author Christian Grothoff
  *
  * @brief performance measurement for the datastore implementation
  * @author Christian Grothoff
  *

- * This testcase inserts a bunch of (variable size) data and then deletes
- * data until the (reported) database size drops below a given threshold.
- * This is iterated 10 times, with the actual size of the content stored,
- * the database size reported and the file size on disk being printed for
- * each iteration.  The code also prints a "I" for every 40 blocks
+ * This testcase inserts a bunch of (variable size) data and then
+ * deletes data until the (reported) database size drops below a given
+ * threshold.  This is iterated 10 times, with the actual size of the
+ * content stored and the number of operations performed being printed
+ * for each iteration.  The code also prints a "I" for every 40 blocks

  * inserted and a "D" for every 40 blocks deleted.  The deletion
  * inserted and a "D" for every 40 blocks deleted.  The deletion

- * strategy alternates between "lowest priority" and "earliest expiration".
- * Priorities and expiration dates are set using a pseudo-random value
- * within a realistic range.
+ * strategy uses the "random" iterator.  Priorities and expiration
+ * dates are set using a pseudo-random value within a realistic range.

  */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
 #include "gnunet_datastore_service.h"
  */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
 #include "gnunet_datastore_service.h"

+#include <gauger.h>
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15)
+
+static const char *plugin_name;

 
 static struct GNUNET_DATASTORE_Handle *datastore;
 
 /**
  * Target datastore size (in bytes).
 
 static struct GNUNET_DATASTORE_Handle *datastore;
 
 /**
  * Target datastore size (in bytes).

- * <p>
- * Example impact of total size on the reported number
- * of operations (insert and delete) per second (once
- * roughly stabilized -- this is not "sound" experimental
- * data but just a rough idea) for a particular machine:
- * <pre>
- *    4: 60   at   7k ops total
- *    8: 50   at   3k ops total
- *   16: 48   at   8k ops total
- *   32: 46   at   8k ops total
- *   64: 61   at   9k ops total
- *  128: 89   at   9k ops total
- * 4092: 11   at 383k ops total (12 GB stored, 14.8 GB DB size on disk, 2.5 GB reported)
- * </pre>
- * Pure insertion performance into an empty DB initially peaks
- * at about 400 ops.  The performance seems to drop especially
- * once the existing (fragmented) ISAM space is filled up and
- * the DB needs to grow on disk.  This could be explained with
- * ISAM looking more carefully for defragmentation opportunities.
- * <p>
- * MySQL disk space overheads (for otherwise unused database when
- * run with 128 MB target data size; actual size 651 MB, useful
- * data stored 520 MB) are quite large in the range of 25-30%.
- * <p>
- * This kind of processing seems to be IO bound (system is roughly
- * at 90% wait, 10% CPU).  This is with MySQL 5.0.
- *

  */
  */

-#define MAX_SIZE 1024LL * 1024 * 16
+#define MAX_SIZE 1024LL * 1024 * 4

 
 /**
  * Report progress outside of major reports? Should probably be GNUNET_YES if
  * size is > 16 MB.
  */
 
 /**
  * Report progress outside of major reports? Should probably be GNUNET_YES if
  * size is > 16 MB.
  */

-#define REPORT_ID GNUNET_NO
-
-/**
- * Number of put operations equivalent to 1/10th of MAX_SIZE
- */
-#define PUT_10 MAX_SIZE / 32 / 1024 / 10
+#define REPORT_ID GNUNET_YES

 
 /**
 
 /**

- * Progress report frequency.  1/10th of a put operation block.
+ * Number of put operations equivalent to 1/3rd of MAX_SIZE

  */
  */

-#define REP_FREQ PUT_10 / 10
+#define PUT_10 MAX_SIZE / 32 / 1024 / 3

 
 /**
  * Total number of iterations (each iteration doing
  * PUT_10 put operations); we report full status every
  * 10 iterations.  Abort with CTRL-C.
  */
 
 /**
  * Total number of iterations (each iteration doing
  * PUT_10 put operations); we report full status every
  * 10 iterations.  Abort with CTRL-C.
  */

-#define ITERATIONS 100
+#define ITERATIONS 8

 
 
 static unsigned long long stored_bytes;
 
 
 static unsigned long long stored_bytes;


@@ -104,113 +81,254 @@ static unsigned long long stored_ops;
 
 static struct GNUNET_TIME_Absolute start_time;
 
 
 static struct GNUNET_TIME_Absolute start_time;
 

-static int
-putValue (int i, int k)
+static int ok;
+
+enum RunPhase

 {
 {

-  size_t size;
-  static GNUNET_HashCode key;
-  static int ic;
-  static char data[65536];
+  RP_DONE = 0,
+  RP_PUT,
+  RP_CUT,
+  RP_REPORT,
+  RP_ERROR
+};
+

 
 

-  /* most content is 32k */
-  size = 32 * 1024;
-  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0)  /* but some of it is less! */
-    size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
-  size = size - (size & 7);     /* always multiple of 8 */
-
-  GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key);
-  memset (data, i, size);
-  if (i > 255)
-    memset (data, i - 255, size / 2);
-  data[0] = k;
-  GNUNET_DATASTORE_put (datastore,
-                       0,
-                       &key,
-                       size,
-                       data,
-                       i,
-                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100),
-                       i,
-                       GNUNET_TIME_relative_to_absolute 
-                       (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
-                                                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
-                       NULL, NULL);
-  ic++;
+struct CpsRunContext
+{
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+  enum RunPhase phase;
+  int j;
+  unsigned long long size;
+  int i;
+};
+
+
+
+static void run_continuation (void *cls,
+                              const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+
+
+static void
+check_success (void *cls, int success, const char *msg)
+{
+  struct CpsRunContext *crc = cls;
+
+  if (GNUNET_OK != success)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Check success failed: `%s'\n", msg);
+    crc->phase = RP_ERROR;
+    GNUNET_SCHEDULER_add_now (&run_continuation, crc);
+    return;
+  }

 #if REPORT_ID
 #if REPORT_ID

-  if (ic % REP_FREQ == 0)
-    fprintf (stderr, "I");
+  fprintf (stderr, "I");

 #endif
 #endif

-  stored_bytes += size;
+  stored_bytes += crc->size;

   stored_ops++;
   stored_entries++;
   stored_ops++;
   stored_entries++;

-  return GNUNET_OK;
+  crc->j++;
+  if (crc->j >= PUT_10)
+  {
+    crc->j = 0;
+    crc->i++;
+    if (crc->i == ITERATIONS)
+      crc->phase = RP_DONE;
+    else
+      crc->phase = RP_CUT;
+  }
+  GNUNET_SCHEDULER_add_continuation (&run_continuation, crc,
+                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);

 }
 
 
 }
 
 

+/**
+ * Continuation called to notify client about result of the
+ * operation.
+ *
+ * @param cls closure
+ * @param success GNUNET_SYSERR on failure
+ * @param msg NULL on success, otherwise an error message
+ */

 static void
 static void

-iterate_delete (void *cls,
-               const GNUNET_HashCode * key,
-               uint32_t size,
-               const void *data,
-               uint32_t type,
-               uint32_t priority,
-               uint32_t anonymity,
-               struct GNUNET_TIME_Absolute
-               expiration, uint64_t uid)
+remove_next (void *cls, int success, const char *msg)

 {
 {

-  GNUNET_DATASTORE_remove (datastore, key, size, data, NULL, NULL);
+  struct CpsRunContext *crc = cls;
+
+  if (GNUNET_OK != success)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "remove_next failed: `%s'\n", msg);
+    crc->phase = RP_ERROR;
+    GNUNET_SCHEDULER_add_now (&run_continuation, crc);
+    return;
+  }
+#if REPORT_ID
+  fprintf (stderr, "D");
+#endif
+  GNUNET_assert (GNUNET_OK == success);
+  GNUNET_SCHEDULER_add_now (&run_continuation, crc);

 }
 
 
 }
 
 

+static void
+delete_value (void *cls, const GNUNET_HashCode * key, size_t size,
+              const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority,
+              uint32_t anonymity, struct GNUNET_TIME_Absolute expiration,
+              uint64_t uid)
+{
+  struct CpsRunContext *crc = cls;
+
+  GNUNET_assert (NULL != key);
+  stored_ops++;
+  stored_bytes -= size;
+  stored_entries--;
+  stored_ops++;
+  if (stored_bytes < MAX_SIZE)
+    crc->phase = RP_PUT;
+  GNUNET_assert (NULL !=
+                 GNUNET_DATASTORE_remove (datastore, key, size, data, 1, 1,
+                                          TIMEOUT, &remove_next, crc));
+}
+

 
 static void
 
 static void

-run (void *cls,
-     struct GNUNET_SCHEDULER_Handle *sched,
-     char *const *args,
-     const char *cfgfile, struct GNUNET_CONFIGURATION_Handle *cfg)
+run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)

 {
 {

-  int j;
-  unsigned long long size;
+  struct CpsRunContext *crc = cls;
+  size_t size;
+  static GNUNET_HashCode key;
+  static char data[65536];

   int i;
   int i;

-
-  datastore = GNUNET_DATASTORE_connect (cfg, sched);
-  /* FIXME: change loop to use CPS; current
-     datastore API will likely react negative to
-     us ignoring the callbacks... */
-  for (i = 0; i < ITERATIONS; i++)
-    {
+  int k;
+  char gstr[128];
+
+  ok = (int) crc->phase;
+  switch (crc->phase)
+  {
+  case RP_PUT:
+    memset (&key, 256 - crc->i, sizeof (GNUNET_HashCode));
+    i = crc->j;
+    k = crc->i;
+    /* most content is 32k */
+    size = 32 * 1024;
+    if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
+      size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
+    crc->size = size = size - (size & 7);       /* always multiple of 8 */
+    GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key);
+    memset (data, i, size);
+    if (i > 255)
+      memset (data, i - 255, size / 2);
+    data[0] = k;
+    GNUNET_assert (NULL !=
+                   GNUNET_DATASTORE_put (datastore, 0, &key, size, data, i + 1,
+                                         GNUNET_CRYPTO_random_u32
+                                         (GNUNET_CRYPTO_QUALITY_WEAK, 100), i,
+                                         0,
+                                         GNUNET_TIME_relative_to_absolute
+                                         (GNUNET_TIME_relative_multiply
+                                          (GNUNET_TIME_UNIT_SECONDS,
+                                           GNUNET_CRYPTO_random_u32
+                                           (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
+                                         1, 1, TIMEOUT, &check_success, crc));
+    break;
+  case RP_CUT:
+    /* trim down below MAX_SIZE again */
+    GNUNET_assert (NULL !=
+                   GNUNET_DATASTORE_get_for_replication (datastore, 1, 1,
+                                                         TIMEOUT, &delete_value,
+                                                         crc));
+    break;
+  case RP_REPORT:
+    printf (

 #if REPORT_ID
 #if REPORT_ID

-      fprintf (stderr, ".");
+             "\n"

 #endif
 #endif

-      /* insert data equivalent to 1/10th of MAX_SIZE */
-      for (j = 0; j < PUT_10; j++)
-       GNUNET_assert (GNUNET_OK == putValue (j, i));
-
-      /* trim down below MAX_SIZE again */
-      if ((i % 2) == 0)
-        GNUNET_DATASTORE_get_random (datastore, &iterate_delete, NULL);
-      size = 0;
-      printf (
-#if REPORT_ID
-               "\n"
-#endif
-               "Stored %llu kB / %lluk ops / %llu ops/s\n", 
-              stored_bytes / 1024,     /* used size in k */
-               (stored_ops * 2 - stored_entries) / 1024,        /* total operations (in k) */
-               1000 * (stored_ops * 2 - stored_entries) / (1 + GNUNET_TIME_absolute_get_duration(start_time).value));       /* operations per second */
-    }
-  GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
+             "Stored %llu kB / %lluk ops / %llu ops/s\n", stored_bytes / 1024,  /* used size in k */
+             stored_ops / 1024, /* total operations (in k) */
+             1000 * stored_ops / (1 +
+                                  GNUNET_TIME_absolute_get_duration
+                                  (start_time).rel_value));
+    crc->phase = RP_PUT;
+    crc->j = 0;
+    GNUNET_SCHEDULER_add_continuation (&run_continuation, crc,
+                                       GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+    break;
+  case RP_DONE:
+    GNUNET_snprintf (gstr, sizeof (gstr), "DATASTORE-%s", plugin_name);
+    if ((crc->i == ITERATIONS) && (stored_ops > 0))
+      GAUGER (gstr, "PUT operation duration",
+              GNUNET_TIME_absolute_get_duration (start_time).rel_value /
+              stored_ops, "ms/operation");
+    GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
+    GNUNET_free (crc);
+    ok = 0;
+    break;
+  case RP_ERROR:
+    GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
+    GNUNET_free (crc);
+    ok = 1;
+    break;
+  default:
+    GNUNET_assert (0);
+  }
+}
+
+
+static void
+run_tests (void *cls, int success, const char *msg)
+{
+  struct CpsRunContext *crc = cls;
+
+  if (success != GNUNET_YES)
+  {
+    fprintf (stderr,
+             "Test 'put' operation failed with error `%s' database likely not setup, skipping test.",
+             msg);
+    GNUNET_free (crc);
+    return;
+  }
+  GNUNET_SCHEDULER_add_continuation (&run_continuation, crc,
+                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+}
+
+
+static void
+run (void *cls, char *const *args, const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  struct CpsRunContext *crc;
+  static GNUNET_HashCode zkey;
+
+  datastore = GNUNET_DATASTORE_connect (cfg);
+  start_time = GNUNET_TIME_absolute_get ();
+  crc = GNUNET_malloc (sizeof (struct CpsRunContext));
+  crc->cfg = cfg;
+  crc->phase = RP_PUT;
+  if (NULL ==
+      GNUNET_DATASTORE_put (datastore, 0, &zkey, 4, "TEST",
+                            GNUNET_BLOCK_TYPE_TEST, 0, 0, 0,
+                            GNUNET_TIME_relative_to_absolute
+                            (GNUNET_TIME_UNIT_SECONDS), 0, 1,
+                            GNUNET_TIME_UNIT_MINUTES, &run_tests, crc))
+  {
+    fprintf (stderr, "Test 'put' operation failed.\n");
+    ok = 1;
+    GNUNET_free (crc);
+  }

 }
 
 
 static int
 check ()
 {
 }
 
 
 static int
 check ()
 {

-  int ok = 1 + 2 + 4 + 8;
-  pid_t pid;
-  char *const argv[] = { "perf-datastore-api",
+  struct GNUNET_OS_Process *proc;
+  char cfg_name[128];
+
+  char *const argv[] = {
+    "perf-datastore-api",

     "-c",
     "-c",

-    "test_datastore_api_data.conf",
+    cfg_name,

 #if VERBOSE
     "-L", "DEBUG",
 #endif
 #if VERBOSE
     "-L", "DEBUG",
 #endif


@@ -219,24 +337,28 @@ check ()
   struct GNUNET_GETOPT_CommandLineOption options[] = {
     GNUNET_GETOPT_OPTION_END
   };
   struct GNUNET_GETOPT_CommandLineOption options[] = {
     GNUNET_GETOPT_OPTION_END
   };

-  pid = GNUNET_OS_start_process ("gnunet-service-datastore",
-                                 "gnunet-service-datastore",
+
+  GNUNET_snprintf (cfg_name, sizeof (cfg_name),
+                   "test_datastore_api_data_%s.conf", plugin_name);
+  proc =
+      GNUNET_OS_start_process (NULL, NULL, "gnunet-service-arm",
+                               "gnunet-service-arm",

 #if VERBOSE
 #if VERBOSE

-                                 "-L", "DEBUG",
+                               "-L", "DEBUG",

 #endif
 #endif

-                                 "-c", "test_datastore_api_data.conf", NULL);
-  sleep (1);
-  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
-                      argv, "perf-datastore-api", "nohelp",
-                      options, &run, &ok);
-  if (0 != PLIBC_KILL (pid, SIGTERM))
-    {
-      GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
-      ok = 1;
-    }
-  GNUNET_OS_process_wait(pid);
-  if (ok != 0)
-    fprintf (stderr, "Missed some testcases: %u\n", ok);
+                               "-c", cfg_name, NULL);
+  GNUNET_assert (NULL != proc);
+  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv,
+                      "perf-datastore-api", "nohelp", options, &run, NULL);
+  sleep (1);                    /* give datastore chance to process 'DROP' */
+  if (0 != GNUNET_OS_process_kill (proc, SIGTERM))
+  {
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
+    ok = 1;
+  }
+  GNUNET_OS_process_wait (proc);
+  GNUNET_OS_process_close (proc);
+  proc = NULL;

   return ok;
 }
 
   return ok;
 }
 


@@ -245,7 +367,22 @@ int
 main (int argc, char *argv[])
 {
   int ret;
 main (int argc, char *argv[])
 {
   int ret;

+  char *pos;
+  char dir_name[128];

 
 

+  sleep (1);
+  /* determine name of plugin to use */
+  plugin_name = argv[0];
+  while (NULL != (pos = strstr (plugin_name, "_")))
+    plugin_name = pos + 1;
+  if (NULL != (pos = strstr (plugin_name, ".")))
+    pos[0] = 0;
+  else
+    pos = (char *) plugin_name;
+
+  GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/test-gnunet-datastore-%s",
+                   plugin_name);
+  GNUNET_DISK_directory_remove (dir_name);

   GNUNET_log_setup ("perf-datastore-api",
 #if VERBOSE
                     "DEBUG",
   GNUNET_log_setup ("perf-datastore-api",
 #if VERBOSE
                     "DEBUG",


@@ -254,9 +391,13 @@ main (int argc, char *argv[])
 #endif
                     NULL);
   ret = check ();
 #endif
                     NULL);
   ret = check ();

-
+  if (pos != plugin_name)
+    pos[0] = '.';
+#if REPORT_ID
+  fprintf (stderr, "\n");
+#endif
+  GNUNET_DISK_directory_remove (dir_name);

   return ret;
 }
 
   return ret;
 }
 

-

 /* end of perf_datastore_api.c */
 /* end of perf_datastore_api.c */