fix datastore quota management -- especially if stats is not working

diff --git a/TODO b/TODO
index 996907c00f4a79ad34e1f9f4650fa1f0cf3feb5b..9c1940a0633e0ad8c3ec0ccaa81de962503552e7 100644 (file)
--- a/TODO
+++ b/TODO
@@ -6,13 +6,8 @@
     sessions even if no connections exist for a while! (likely related...)
   - transport reports bw quota violations on slow systems (core or
     transport issue? tracking issue or actual violation?)
-* DATASTORE: [CG]
-  - utilization can (easily, restart?) go out of control (very large), causing
-    content expiration job to go crazy and delete everything!
 * FS: [CG]
-  - on some systems, keyword search does not find locally published content
-    (need testcase of command-line tools! - also good to cover getopt API!)
-    [could be related to datastore issue above!]
+  - Need testcase of command-line tools! - also good to cover getopt API!
   - 2-peer download is still too slow (why? testcases fail on slow systems!)
 * WWW:
   - Get IPv6 hooked up [AK, after April 12th]

diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index c3370e0a582179a53787ab580c36c599adb9efcb..9b23f61054577242f671588398c64c736b673591 100644 (file)
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -297,7 +297,6 @@ expired_processor (void *cls,
 {
   struct GNUNET_TIME_Absolute now;
 
-  expired_kill_task = GNUNET_SCHEDULER_NO_TASK;
   if (key == NULL) 
     {
       expired_kill_task 
@@ -343,6 +342,7 @@ static void
 delete_expired (void *cls,
                const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
+  expired_kill_task = GNUNET_SCHEDULER_NO_TASK;
   plugin->api->iter_ascending_expiration (plugin->api->cls, 
                                          0,
                                          &expired_processor,

diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index 1d38f8326ca2e929c4b85020c0dd5c3d6b0f83b8..cecdf094f6ff0126710cc0b5ce3a55f4d8e802da 100644 (file)
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -29,7 +29,7 @@
 #include "plugin_datastore.h"
 #include <sqlite3.h>
 
-#define DEBUG_SQLITE GNUNET_NO
+#define DEBUG_SQLITE GNUNET_YES
 
 /**
  * After how many payload-changing operations
@@ -159,6 +159,11 @@ struct Plugin
    * Should the database be dropped on shutdown?
    */
   int drop_on_shutdown;
+
+  /**
+   * Did we get an answer from statistics?
+   */
+  int stats_worked;
 };
 
 
@@ -283,11 +288,15 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
   CHECK (SQLITE_OK ==
          sqlite3_exec (plugin->dbh,
                        "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
+  CHECK (SQLITE_OK ==
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
   CHECK (SQLITE_OK ==
          sqlite3_exec (plugin->dbh,
                        "PRAGMA count_changes=OFF", NULL, NULL, ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, ENULL));
+         sqlite3_exec (plugin->dbh, 
+                      "PRAGMA page_size=4092", NULL, NULL, ENULL));
 
   CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
 
@@ -642,7 +651,11 @@ sqlite_next_request_cont (void *cls,
   if ( (ret == GNUNET_NO) &&
        (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
     {
-      plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
+      if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD)
+       plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
+      else
+       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                   _("Datastore payload inaccurate, please fix and restart!\n"));
       plugin->lastSync++; 
 #if DEBUG_SQLITE
       if (ret == GNUNET_NO)
@@ -651,7 +664,7 @@ sqlite_next_request_cont (void *cls,
                         "Removed entry %llu (%u bytes), new payload is %llu\n",
                         (unsigned long long) rowid,
                         size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
-                        (unsigned long long) plugin->payload);
+                        plugin->payload);
 #endif
       if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
        sync_stats (plugin);
@@ -779,7 +792,7 @@ sqlite_plugin_put (void *cls,
                   "sqlite",
                   "Stored new entry (%u bytes), new payload is %llu\n",
                   size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
-                  (unsigned long long) plugin->payload);
+                  plugin->payload);
 #endif
   if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
     sync_stats (plugin);
@@ -1566,13 +1579,15 @@ process_stat_in (void *cls,
                 int is_persistent)
 {
   struct Plugin *plugin = cls;
+
+  plugin->stats_worked = GNUNET_YES;
   plugin->payload += value;
 #if DEBUG_SQLITE
   GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                   "sqlite",
                   "Notification from statistics about existing payload (%llu), new payload is %llu\n",
                   value,
-                  (unsigned long long) plugin->payload);
+                  plugin->payload);
 #endif
   return GNUNET_OK;
 }
@@ -1583,7 +1598,35 @@ process_stat_done (void *cls,
                   int success)
 {
   struct Plugin *plugin = cls;
+  sqlite3_stmt *stmt;
+  uint64_t pages;
+  uint64_t free_pages;
+
   plugin->stat_get = NULL;
+  if (plugin->stats_worked == GNUNET_NO)
+    {
+      CHECK (SQLITE_OK ==
+            sq_prepare (plugin->dbh,
+                        "PRAGMA page_count",
+                        &stmt));
+      CHECK (SQLITE_ROW ==
+            sqlite3_step (stmt));
+      pages = sqlite3_column_int64 (stmt, 0);
+      sqlite3_finalize (stmt);
+      CHECK (SQLITE_OK ==
+            sq_prepare (plugin->dbh,
+                        "PRAGMA freelist_count",
+                        &stmt));
+      CHECK (SQLITE_ROW ==
+            sqlite3_step (stmt));
+      free_pages = sqlite3_column_int64 (stmt, 0);
+      sqlite3_finalize (stmt);
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                 _("Using sqlite page utilization to estimate payload (%llu pages total, %llu free)\n"),
+                 (unsigned long long) pages,
+                 (unsigned long long) free_pages);
+      plugin->payload = (pages - free_pages) * 4092LL;
+    }
 }
                                         
 
@@ -1610,7 +1653,7 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics,
                                           "datastore-sqlite",
                                           QUOTA_STAT_NAME,
-                                          GNUNET_TIME_UNIT_MINUTES,
+                                          GNUNET_TIME_UNIT_SECONDS,
                                           &process_stat_done,
                                           &process_stat_in,
                                           &plugin);
@@ -1635,6 +1678,9 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   api->drop = &sqlite_plugin_drop;
   GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
                    "sqlite", _("Sqlite database running\n"));
+  fprintf (stderr,
+          "Initial payload: %llu\n",
+          plugin.payload);
   return api;
 }