-removed obsolete functions

[oweals/gnunet.git] / src / fs / gnunet-service-fs_indexing.c

diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
index f8593ffd45e846ed1a9c632b27a56e280a1bc359..0200cb9d41354a02da7f1c8a3492489d1f867d61 100644 (file)
--- a/src/fs/gnunet-service-fs_indexing.c
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -1,10 +1,10 @@
 /*
      This file is part of GNUnet.
 /*
      This file is part of GNUnet.

-     (C) 2009, 2010 Christian Grothoff (and other contributing authors)
+     Copyright (C) 2009, 2010 Christian Grothoff (and other contributing authors)

 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published

-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your

      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but


@@ -14,19 +14,14 @@
 
      You should have received a copy of the GNU General Public License
      along with GNUnet; see the file COPYING.  If not, write to the
 
      You should have received a copy of the GNU General Public License
      along with GNUnet; see the file COPYING.  If not, write to the

-     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-     Boston, MA 02111-1307, USA.
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.

 */
 
 /**
  * @file fs/gnunet-service-fs_indexing.c
  * @brief program that provides indexing functions of the file-sharing service
  * @author Christian Grothoff
 */
 
 /**
  * @file fs/gnunet-service-fs_indexing.c
  * @brief program that provides indexing functions of the file-sharing service
  * @author Christian Grothoff

- *
- * TODO:
- * - consider doing GNUNET_CRYPTO_hash_file_cancel on active indexing
- *   jobs during shutdown (currently, shutdown will only happen after
- *   all of those are done, not sure if this is good or bad)

  */
 #include "platform.h"
 #include <float.h>
  */
 #include "platform.h"
 #include <float.h>


@@ -36,24 +31,27 @@
 #include "gnunet_protocols.h"
 #include "gnunet_signatures.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
 #include "gnunet_signatures.h"
 #include "gnunet_util_lib.h"

-#include "gnunet-service-fs_drq.h"
+#include "gnunet-service-fs.h"

 #include "gnunet-service-fs_indexing.h"
 #include "fs.h"
 
 #include "gnunet-service-fs_indexing.h"
 #include "fs.h"
 

-#define DEBUG_FS GNUNET_YES
-

 /**
  * In-memory information about indexed files (also available
  * on-disk).
  */
 struct IndexInfo
 {
 /**
  * In-memory information about indexed files (also available
  * on-disk).
  */
 struct IndexInfo
 {

-  
+

   /**
   /**

-   * This is a linked list.
+   * This is a doubly linked list.

    */
   struct IndexInfo *next;
 
    */
   struct IndexInfo *next;
 

+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexInfo *prev;
+

   /**
    * Name of the indexed file.  Memory allocated
    * at the end of this struct (do not free).
   /**
    * Name of the indexed file.  Memory allocated
    * at the end of this struct (do not free).


@@ -70,91 +68,81 @@ struct IndexInfo
    * Context for hashing of the file.
    */
   struct GNUNET_CRYPTO_FileHashContext *fhc;
    * Context for hashing of the file.
    */
   struct GNUNET_CRYPTO_FileHashContext *fhc;

-  
+

   /**
    * Hash of the contents of the file.
    */
   /**
    * Hash of the contents of the file.
    */

-  GNUNET_HashCode file_id;
+  struct GNUNET_HashCode file_id;

 
 };
 
 
 /**
 
 };
 
 
 /**

- * Linked list of indexed files.
+ * Head of linked list of indexed files.

  */
  */

-static struct IndexInfo *indexed_files;
+static struct IndexInfo *indexed_files_head;

 
 /**
 
 /**

- * Maps hash over content of indexed files to the respective filename.
+ * Tail of linked list of indexed files.
+ */
+static struct IndexInfo *indexed_files_tail;
+
+/**
+ * Maps hash over content of indexed files to the respective 'struct IndexInfo'.

  * The filenames are pointers into the indexed_files linked list and
  * do not need to be freed.
  */
 static struct GNUNET_CONTAINER_MultiHashMap *ifm;
 
 /**
  * The filenames are pointers into the indexed_files linked list and
  * do not need to be freed.
  */
 static struct GNUNET_CONTAINER_MultiHashMap *ifm;
 
 /**

- * Our scheduler.
+ * Our configuration.

  */
  */

-static struct GNUNET_SCHEDULER_Handle *sched;
+static const struct GNUNET_CONFIGURATION_Handle *cfg;

 
 /**
 
 /**

- * Our configuration.
+ * Datastore handle.  Created and destroyed by code in
+ * gnunet-service-fs (this is an alias).

  */
  */

-static const struct GNUNET_CONFIGURATION_Handle *cfg;
+static struct GNUNET_DATASTORE_Handle *dsh;

 
 
 /**
  * Write the current index information list to disk.
 
 
 /**
  * Write the current index information list to disk.

- */ 
+ */

 static void
 write_index_list ()
 {
   struct GNUNET_BIO_WriteHandle *wh;
   char *fn;
 static void
 write_index_list ()
 {
   struct GNUNET_BIO_WriteHandle *wh;
   char *fn;

-  struct IndexInfo *pos;  
+  struct IndexInfo *pos;

 
   if (GNUNET_OK !=
 
   if (GNUNET_OK !=

-      GNUNET_CONFIGURATION_get_value_filename (cfg,
-                                              "FS",
-                                              "INDEXDB",
-                                              &fn))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                 _("Configuration option `%s' in section `%s' missing.\n"),
-                 "INDEXDB",
-                 "FS");
-      return;
-    }
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                              "fs", "INDEXDB");
+    return;
+  }

   wh = GNUNET_BIO_write_open (fn);
   if (NULL == wh)
   wh = GNUNET_BIO_write_open (fn);
   if (NULL == wh)

-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                 _("Could not open `%s'.\n"),
-                 fn);
-      GNUNET_free (fn);
-      return;
-    }
-  pos = indexed_files;
-  while (pos != NULL)
-    {
-      if ( (GNUNET_OK !=
-           GNUNET_BIO_write (wh,
-                             &pos->file_id,
-                             sizeof (GNUNET_HashCode))) ||
-          (GNUNET_OK !=
-           GNUNET_BIO_write_string (wh,
-                                    pos->filename)) )
-       break;
-      pos = pos->next;
-    }
-  if (GNUNET_OK != 
-      GNUNET_BIO_write_close (wh))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                 _("Error writing `%s'.\n"),
-                 fn);
-      GNUNET_free (fn);
-      return;
-    }
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _("Could not open `%s'.\n"), fn);
+    GNUNET_free (fn);
+    return;
+  }
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+    if ((GNUNET_OK !=
+         GNUNET_BIO_write (wh, &pos->file_id, sizeof (struct GNUNET_HashCode))) ||
+        (GNUNET_OK != GNUNET_BIO_write_string (wh, pos->filename)))
+      break;
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _("Error writing `%s'.\n"), fn);
+    GNUNET_free (fn);
+    return;
+  }

   GNUNET_free (fn);
 }
 
   GNUNET_free (fn);
 }
 


@@ -167,73 +155,60 @@ read_index_list ()
 {
   struct GNUNET_BIO_ReadHandle *rh;
   char *fn;
 {
   struct GNUNET_BIO_ReadHandle *rh;
   char *fn;

-  struct IndexInfo *pos;  
+  struct IndexInfo *pos;

   char *fname;
   char *fname;

-  GNUNET_HashCode hc;
+  struct GNUNET_HashCode hc;

   size_t slen;
   char *emsg;
 
   if (GNUNET_OK !=
   size_t slen;
   char *emsg;
 
   if (GNUNET_OK !=

-      GNUNET_CONFIGURATION_get_value_filename (cfg,
-                                              "FS",
-                                              "INDEXDB",
-                                              &fn))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                 _("Configuration option `%s' in section `%s' missing.\n"),
-                 "INDEXDB",
-                 "FS");
-      return;
-    }
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                              "fs", "INDEXDB");
+    return;
+  }

   if (GNUNET_NO == GNUNET_DISK_file_test (fn))
   if (GNUNET_NO == GNUNET_DISK_file_test (fn))

-    {
-      /* no index info yet */
-      GNUNET_free (fn);
-      return;
-    }
+  {
+    /* no index info yet */
+    GNUNET_free (fn);
+    return;
+  }

   rh = GNUNET_BIO_read_open (fn);
   if (NULL == rh)
   rh = GNUNET_BIO_read_open (fn);
   if (NULL == rh)

+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _("Could not open `%s'.\n"), fn);
+    GNUNET_free (fn);
+    return;
+  }
+  while ((GNUNET_OK ==
+          GNUNET_BIO_read (rh, "Hash of indexed file", &hc,
+                           sizeof (struct GNUNET_HashCode))) &&
+         (GNUNET_OK ==
+          GNUNET_BIO_read_string (rh, "Name of indexed file", &fname,
+                                  1024 * 16)) && (fname != NULL))
+  {
+    slen = strlen (fname) + 1;
+    pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
+    pos->file_id = hc;
+    pos->filename = (const char *) &pos[1];
+    memcpy (&pos[1], fname, slen);
+    if (GNUNET_SYSERR ==
+        GNUNET_CONTAINER_multihashmap_put (ifm, &pos->file_id, pos,
+                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))

     {
     {

-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                 _("Could not open `%s'.\n"),
-                 fn);
-      GNUNET_free (fn);
-      return;
+      GNUNET_free (pos);

     }
     }

-
-  while ( (GNUNET_OK ==
-          GNUNET_BIO_read (rh,
-                           "Hash of indexed file",
-                           &hc,
-                           sizeof (GNUNET_HashCode))) &&
-         (GNUNET_OK ==
-          GNUNET_BIO_read_string (rh, 
-                                  "Name of indexed file",
-                                  &fname,
-                                  1024 * 16)) )
+    else

     {
     {

-      slen = strlen (fname) + 1;
-      pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
-      pos->file_id = hc;
-      pos->filename = (const char *) &pos[1];
-      memcpy (&pos[1], fname, slen);
-      if (GNUNET_SYSERR ==
-         GNUNET_CONTAINER_multihashmap_put (ifm,
-                                            &hc,
-                                            (void*) pos->filename,
-                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-       {
-         GNUNET_free (pos);
-       }
-      else
-       {
-         pos->next = indexed_files;
-         indexed_files = pos;
-       }
-      GNUNET_free (fname);
+      GNUNET_CONTAINER_DLL_insert (indexed_files_head,
+                                  indexed_files_tail,
+                                  pos);

     }
     }

-  if (GNUNET_OK != 
-      GNUNET_BIO_read_close (rh, &emsg))
+    GNUNET_free (fname);
+  }
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))

     GNUNET_free (emsg);
   GNUNET_free (fn);
 }
     GNUNET_free (emsg);
   GNUNET_free (fn);
 }


@@ -248,33 +223,33 @@ read_index_list ()
 static void
 signal_index_ok (struct IndexInfo *ii)
 {
 static void
 signal_index_ok (struct IndexInfo *ii)
 {

+  struct IndexInfo *ir;

   if (GNUNET_SYSERR ==
   if (GNUNET_SYSERR ==

-      GNUNET_CONTAINER_multihashmap_put (ifm,
-                                        &ii->file_id,
-                                        (void*) ii->filename,
-                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                 _("Index request received for file `%s' is already indexed as `%s'.  Permitting anyway.\n"),
-                 ii->filename,
-                 (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
-                                                                  &ii->file_id));
-      GNUNET_SERVER_transmit_context_append_data (ii->tc,
-                                                 NULL, 0,
-                                                 GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
-      GNUNET_SERVER_transmit_context_run (ii->tc,
-                                         GNUNET_TIME_UNIT_MINUTES);
-      GNUNET_free (ii);
-      return;
-    }
-  ii->next = indexed_files;
-  indexed_files = ii;
+      GNUNET_CONTAINER_multihashmap_put (ifm, &ii->file_id,
+                                         ii,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  {
+    ir = GNUNET_CONTAINER_multihashmap_get (ifm,
+                                           &ii->file_id);
+    GNUNET_assert (NULL != ir);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _
+                ("Index request received for file `%s' is already indexed as `%s'.  Permitting anyway.\n"),
+                ii->filename,
+               ir->filename);
+    GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0,
+                                                GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
+    GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES);
+    GNUNET_free (ii);
+    return;
+  }
+  GNUNET_CONTAINER_DLL_insert (indexed_files_head,
+                              indexed_files_tail,
+                              ii);

   write_index_list ();
   write_index_list ();

-  GNUNET_SERVER_transmit_context_append_data (ii->tc,
-                                             NULL, 0,
-                                             GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
-  GNUNET_SERVER_transmit_context_run (ii->tc,
-                                     GNUNET_TIME_UNIT_MINUTES);
+  GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0,
+                                              GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
+  GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES);

   ii->tc = NULL;
 }
 
   ii->tc = NULL;
 }
 


@@ -286,36 +261,27 @@ signal_index_ok (struct IndexInfo *ii)
  * @param cls closure, our publishing context
  * @param res resulting hash, NULL on error
  */
  * @param cls closure, our publishing context
  * @param res resulting hash, NULL on error
  */

-static void 
-hash_for_index_val (void *cls,
-                   const GNUNET_HashCode *
-                   res)
+static void
+hash_for_index_val (void *cls, const struct GNUNET_HashCode * res)

 {
   struct IndexInfo *ii = cls;
 
   ii->fhc = NULL;
 {
   struct IndexInfo *ii = cls;
 
   ii->fhc = NULL;

-  if ( (res == NULL) ||
-       (0 != memcmp (res,
-                    &ii->file_id,
-                    sizeof(GNUNET_HashCode))) )
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
-                 ii->filename,
-                 GNUNET_h2s (res));
-#if DEBUG_FS
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Wanted `%s'\n",
-                 GNUNET_h2s (&ii->file_id));
-#endif
-      GNUNET_SERVER_transmit_context_append_data (ii->tc,
-                                                 NULL, 0,
-                                                 GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
-      GNUNET_SERVER_transmit_context_run (ii->tc,
-                                         GNUNET_TIME_UNIT_MINUTES);
-      GNUNET_free (ii);
-      return;
-    }
+  if ((res == NULL) ||
+      (0 != memcmp (res, &ii->file_id, sizeof (struct GNUNET_HashCode))))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _
+                ("Hash mismatch trying to index file `%s' which has hash `%s'\n"),
+                ii->filename, GNUNET_h2s (res));
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Wanted `%s'\n",
+                GNUNET_h2s (&ii->file_id));
+    GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0,
+                                                GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
+    GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES);
+    GNUNET_free (ii);
+    return;
+  }

   signal_index_ok (ii);
 }
 
   signal_index_ok (ii);
 }
 


@@ -328,72 +294,72 @@ hash_for_index_val (void *cls,
  * @param message the actual message
  */
 void
  * @param message the actual message
  */
 void

-GNUNET_FS_handle_index_start (void *cls,
-                             struct GNUNET_SERVER_Client *client,
-                             const struct GNUNET_MessageHeader *message)
+GNUNET_FS_handle_index_start (void *cls, struct GNUNET_SERVER_Client *client,
+                              const struct GNUNET_MessageHeader *message)

 {
   const struct IndexStartMessage *ism;
   char *fn;
   uint16_t msize;
   struct IndexInfo *ii;
   size_t slen;
 {
   const struct IndexStartMessage *ism;
   char *fn;
   uint16_t msize;
   struct IndexInfo *ii;
   size_t slen;

-  uint32_t dev;
+  uint64_t dev;

   uint64_t ino;
   uint64_t ino;

-  uint32_t mydev;
+  uint64_t mydev;

   uint64_t myino;
 
   uint64_t myino;
 

-  msize = ntohs(message->size);
-  if ( (msize <= sizeof (struct IndexStartMessage)) ||
-       ( ((const char *)message)[msize-1] != '\0') )
-    {
-      GNUNET_break (0);
-      GNUNET_SERVER_receive_done (client,
-                                 GNUNET_SYSERR);
-      return;
-    }
-  ism = (const struct IndexStartMessage*) message;
-  fn = GNUNET_STRINGS_filename_expand ((const char*) &ism[1]);
-  dev = ntohl (ism->device);
+  msize = ntohs (message->size);
+  if ((msize <= sizeof (struct IndexStartMessage)) ||
+      (((const char *) message)[msize - 1] != '\0'))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return;
+  }
+  ism = (const struct IndexStartMessage *) message;
+  if (0 != ism->reserved)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return;
+  }
+  fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]);
+  if (fn == NULL)
+  {
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return;
+  }
+  dev = GNUNET_ntohll (ism->device);

   ino = GNUNET_ntohll (ism->inode);
   ino = GNUNET_ntohll (ism->inode);

-  ism = (const struct IndexStartMessage*) message;
+  ism = (const struct IndexStartMessage *) message;

   slen = strlen (fn) + 1;
   ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
   slen = strlen (fn) + 1;
   ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);

-  ii->filename = (const char*) &ii[1];
+  ii->filename = (const char *) &ii[1];

   memcpy (&ii[1], fn, slen);
   memcpy (&ii[1], fn, slen);

-  ii->file_id = ism->file_id;  
+  ii->file_id = ism->file_id;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received `%s' message for file `%s'\n",
+              "START_INDEX", ii->filename);

   ii->tc = GNUNET_SERVER_transmit_context_create (client);
   mydev = 0;
   myino = 0;
   ii->tc = GNUNET_SERVER_transmit_context_create (client);
   mydev = 0;
   myino = 0;

-  if ( ( (dev != 0) ||
-        (ino != 0) ) &&
-       (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
-                                                      &mydev,
-                                                      &myino)) &&
-       ( (dev == mydev) &&
-        (ino == myino) ) )
-    {      
-      /* fast validation OK! */
-      signal_index_ok (ii);
-      GNUNET_free (fn);
-      return;
-    }
-#if DEBUG_FS
+  if (((dev != 0) || (ino != 0)) &&
+      (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn, &mydev, &myino)) &&
+      ((dev == mydev) && (ino == myino)))
+  {
+    /* fast validation OK! */
+    signal_index_ok (ii);
+    GNUNET_free (fn);
+    return;
+  }

   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,

-             "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
-             (unsigned long long) ino,
-             (unsigned long long) myino,
-             (unsigned int) dev,
-             (unsigned int) mydev);
-#endif
+              "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
+              (unsigned long long) ino, (unsigned long long) myino,
+              (unsigned int) dev, (unsigned int) mydev);

   /* slow validation, need to hash full file (again) */
   /* slow validation, need to hash full file (again) */

-  ii->fhc = GNUNET_CRYPTO_hash_file (sched,
-                                    GNUNET_SCHEDULER_PRIORITY_IDLE,
-                                    fn,
-                                    HASHING_BLOCKSIZE,
-                                    &hash_for_index_val,
-                                    ii);
-  if (ii->fhc == NULL)    
-    hash_for_index_val (ii, NULL);    
+  ii->fhc =
+      GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, fn,
+                               HASHING_BLOCKSIZE, &hash_for_index_val, ii);
+  if (ii->fhc == NULL)
+    hash_for_index_val (ii, NULL);

   GNUNET_free (fn);
 }
 
   GNUNET_free (fn);
 }
 


@@ -406,44 +372,38 @@ GNUNET_FS_handle_index_start (void *cls,
  * @param message the actual message
  */
 void
  * @param message the actual message
  */
 void

-GNUNET_FS_handle_index_list_get (void *cls,
-                                struct GNUNET_SERVER_Client *client,
-                                const struct GNUNET_MessageHeader *message)
+GNUNET_FS_handle_index_list_get (void *cls, struct GNUNET_SERVER_Client *client,
+                                 const struct GNUNET_MessageHeader *message)

 {
   struct GNUNET_SERVER_TransmitContext *tc;
   struct IndexInfoMessage *iim;
 {
   struct GNUNET_SERVER_TransmitContext *tc;
   struct IndexInfoMessage *iim;

-  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
+  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;

   size_t slen;
   const char *fn;
   struct IndexInfo *pos;
 
   tc = GNUNET_SERVER_transmit_context_create (client);
   size_t slen;
   const char *fn;
   struct IndexInfo *pos;
 
   tc = GNUNET_SERVER_transmit_context_create (client);

-  iim = (struct IndexInfoMessage*) buf;
-  pos = indexed_files;
-  while (NULL != pos)
+  iim = (struct IndexInfoMessage *) buf;
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+  {
+    fn = pos->filename;
+    slen = strlen (fn) + 1;
+    if (slen + sizeof (struct IndexInfoMessage) >=
+        GNUNET_SERVER_MAX_MESSAGE_SIZE)

     {
     {

-      fn = pos->filename;
-      slen = strlen (fn) + 1;
-      if (slen + sizeof (struct IndexInfoMessage) > 
-         GNUNET_SERVER_MAX_MESSAGE_SIZE)
-       {
-         GNUNET_break (0);
-         break;
-       }
-      iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
-      iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
-      iim->reserved = 0;
-      iim->file_id = pos->file_id;
-      memcpy (&iim[1], fn, slen);
-      GNUNET_SERVER_transmit_context_append_message (tc,
-                                                    &iim->header);
-      pos = pos->next;
+      GNUNET_break (0);
+      break;

     }
     }

-  GNUNET_SERVER_transmit_context_append_data (tc,
-                                             NULL, 0,
-                                             GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
-  GNUNET_SERVER_transmit_context_run (tc,
-                                     GNUNET_TIME_UNIT_MINUTES);
+    iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
+    iim->header.size = htons (slen + sizeof (struct IndexInfoMessage));
+    iim->reserved = 0;
+    iim->file_id = pos->file_id;
+    memcpy (&iim[1], fn, slen);
+    GNUNET_SERVER_transmit_context_append_message (tc, &iim->header);
+  }
+  GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0,
+                                              GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
+  GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_MINUTES);

 }
 
 
 }
 
 


@@ -455,77 +415,66 @@ GNUNET_FS_handle_index_list_get (void *cls,
  * @param message the actual message
  */
 void
  * @param message the actual message
  */
 void

-GNUNET_FS_handle_unindex (void *cls,
-                         struct GNUNET_SERVER_Client *client,
-                         const struct GNUNET_MessageHeader *message)
+GNUNET_FS_handle_unindex (void *cls, struct GNUNET_SERVER_Client *client,
+                          const struct GNUNET_MessageHeader *message)

 {
   const struct UnindexMessage *um;
   struct IndexInfo *pos;
 {
   const struct UnindexMessage *um;
   struct IndexInfo *pos;

-  struct IndexInfo *prev;
-  struct IndexInfo *next;

   struct GNUNET_SERVER_TransmitContext *tc;
   int found;
   struct GNUNET_SERVER_TransmitContext *tc;
   int found;

-  
-  um = (const struct UnindexMessage*) message;
+
+  um = (const struct UnindexMessage *) message;
+  if (0 != um->reserved)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return;
+  }

   found = GNUNET_NO;
   found = GNUNET_NO;

-  prev = NULL;
-  pos = indexed_files;
-  while (NULL != pos)
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+  {
+    if (0 == memcmp (&pos->file_id, &um->file_id, sizeof (struct GNUNET_HashCode)))

     {
     {

-      next = pos->next;
-      if (0 == memcmp (&pos->file_id,
-                      &um->file_id,
-                      sizeof (GNUNET_HashCode)))
-       {
-         if (prev == NULL)
-           indexed_files = next;
-         else
-           prev->next = next;
-         GNUNET_free (pos);
-         found = GNUNET_YES;
-       }
-      else
-       {
-         prev = pos;
-       }
-      pos = next;
+      GNUNET_CONTAINER_DLL_remove (indexed_files_head,
+                                  indexed_files_tail,
+                                  pos);
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id,
+                                                         pos));
+      GNUNET_free (pos);
+      found = GNUNET_YES;
+      break;

     }
     }

-#if DEBUG_FS
+  }

   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,

-             "Client requested unindexing of file `%s': %s\n",
-             GNUNET_h2s (&um->file_id),
-             found ? "found" : "not found");
-#endif
-  if (GNUNET_YES == found)    
+              "Client requested unindexing of file `%s': %s\n",
+              GNUNET_h2s (&um->file_id), found ? "found" : "not found");
+  if (GNUNET_YES == found)

     write_index_list ();
   tc = GNUNET_SERVER_transmit_context_create (client);
     write_index_list ();
   tc = GNUNET_SERVER_transmit_context_create (client);

-  GNUNET_SERVER_transmit_context_append_data (tc,
-                                             NULL, 0,
-                                             GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
-  GNUNET_SERVER_transmit_context_run (tc,
-                                     GNUNET_TIME_UNIT_MINUTES);
+  GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0,
+                                              GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
+  GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_MINUTES);

 }
 
 
 }
 
 

-
-

 /**
  * Continuation called from datastore's remove
  * function.
  *
  * @param cls unused
  * @param success did the deletion work?
 /**
  * Continuation called from datastore's remove
  * function.
  *
  * @param cls unused
  * @param success did the deletion work?

+ * @param min_expiration minimum expiration time required for content to be stored

  * @param msg error message
  */
 static void
  * @param msg error message
  */
 static void

-remove_cont (void *cls,
-            int success,
+remove_cont (void *cls, int success,
+            struct GNUNET_TIME_Absolute min_expiration,

             const char *msg)
 {
   if (GNUNET_OK != success)
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
             const char *msg)
 {
   if (GNUNET_OK != success)
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,

-               _("Failed to delete bogus block: %s\n"),
-               msg);
+                _("Failed to delete bogus block: %s\n"), msg);

 }
 
 
 }
 
 


@@ -549,140 +498,115 @@ remove_cont (void *cls,
  * @return GNUNET_OK on success
  */
 int
  * @return GNUNET_OK on success
  */
 int

-GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key,
-                                 uint32_t size,
-                                 const void *data,
-                                 enum GNUNET_BLOCK_Type type,
-                                 uint32_t priority,
-                                 uint32_t anonymity,
-                                 struct GNUNET_TIME_Absolute
-                                 expiration, uint64_t uid,
-                                 GNUNET_DATASTORE_Iterator cont,
-                                 void *cont_cls)
+GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode * key, uint32_t size,
+                                  const void *data, enum GNUNET_BLOCK_Type type,
+                                  uint32_t priority, uint32_t anonymity,
+                                  struct GNUNET_TIME_Absolute expiration,
+                                  uint64_t uid,
+                                  GNUNET_DATASTORE_DatumProcessor cont,
+                                  void *cont_cls)

 {
   const struct OnDemandBlock *odb;
 {
   const struct OnDemandBlock *odb;

-  GNUNET_HashCode nkey;
-  struct GNUNET_CRYPTO_AesSessionKey skey;
-  struct GNUNET_CRYPTO_AesInitializationVector iv;
-  GNUNET_HashCode query;
+  struct GNUNET_HashCode nkey;
+  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_HashCode query;

   ssize_t nsize;
   char ndata[DBLOCK_SIZE];
   char edata[DBLOCK_SIZE];
   const char *fn;
   struct GNUNET_DISK_FileHandle *fh;
   uint64_t off;
   ssize_t nsize;
   char ndata[DBLOCK_SIZE];
   char edata[DBLOCK_SIZE];
   const char *fn;
   struct GNUNET_DISK_FileHandle *fh;
   uint64_t off;

+  struct IndexInfo *ii;

 
   if (size != sizeof (struct OnDemandBlock))
 
   if (size != sizeof (struct OnDemandBlock))

-    {
-      GNUNET_break (0);
-      GNUNET_FS_drq_remove (key,
-                           size,
-                           data,
-                           &remove_cont,
-                           NULL,
-                           GNUNET_TIME_UNIT_FOREVER_REL);
-      return GNUNET_SYSERR;
-    }
-  odb = (const struct OnDemandBlock*) data;
+  {
+    GNUNET_break (0);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1,
+                             GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  odb = (const struct OnDemandBlock *) data;

   off = GNUNET_ntohll (odb->offset);
   off = GNUNET_ntohll (odb->offset);

-  fn = (const char*) GNUNET_CONTAINER_multihashmap_get (ifm,
-                                                       &odb->file_id);
-  fh = NULL;
-  if ( (NULL == fn) ||
-       (NULL == (fh = GNUNET_DISK_file_open (fn, 
-                                            GNUNET_DISK_OPEN_READ,
-                                            GNUNET_DISK_PERM_NONE))) ||
-       (off !=
-       GNUNET_DISK_file_seek (fh,
-                              off,
-                              GNUNET_DISK_SEEK_SET)) ||
-       (-1 ==
-       (nsize = GNUNET_DISK_file_read (fh,
-                                       ndata,
-                                       sizeof (ndata)))) )
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("Could not access indexed file `%s' at offset %llu: %s\n"),
-                 GNUNET_h2s (&odb->file_id),
-                 (unsigned long long) off,
-                 STRERROR (errno));
-      if (fh != NULL)
-       GNUNET_DISK_file_close (fh);
-      GNUNET_FS_drq_remove (key,
-                           size,
-                           data,
-                           &remove_cont,
-                           NULL,
-                           GNUNET_TIME_UNIT_FOREVER_REL);
-      return GNUNET_SYSERR;
-    }
+  ii = GNUNET_CONTAINER_multihashmap_get (ifm, &odb->file_id);
+  if (NULL == ii)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  fn = ii->filename;
+  if ((NULL == fn) || (0 != ACCESS (fn, R_OK)))
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# index blocks removed: original file inaccessible"),
+                              1, GNUNET_YES);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1,
+                             GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  if ((NULL ==
+       (fh =
+        GNUNET_DISK_file_open (fn, GNUNET_DISK_OPEN_READ,
+                               GNUNET_DISK_PERM_NONE))) ||
+      (off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)) ||
+      (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof (ndata)))))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _
+                ("Could not access indexed file `%s' (%s) at offset %llu: %s\n"),
+                GNUNET_h2s (&odb->file_id), fn, (unsigned long long) off,
+                (fn == NULL) ? _("not indexed") : STRERROR (errno));
+    if (fh != NULL)
+      GNUNET_DISK_file_close (fh);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1,
+                             GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }

   GNUNET_DISK_file_close (fh);
   GNUNET_DISK_file_close (fh);

-  GNUNET_CRYPTO_hash (ndata,
-                     nsize,
-                     &nkey);
+  GNUNET_CRYPTO_hash (ndata, nsize, &nkey);

   GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
   GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);

-  GNUNET_CRYPTO_aes_encrypt (ndata,
-                            nsize,
-                            &skey,
-                            &iv,
-                            edata);
-  GNUNET_CRYPTO_hash (edata,
-                     nsize,
-                     &query);
-  if (0 != memcmp (&query, 
-                  key,
-                  sizeof (GNUNET_HashCode)))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("Indexed file `%s' changed at offset %llu\n"),
-                 fn,
-                 (unsigned long long) off);
-      GNUNET_FS_drq_remove (key,
-                           size,
-                           data,
-                           &remove_cont,
-                           NULL,
-                           GNUNET_TIME_UNIT_FOREVER_REL);
-      return GNUNET_SYSERR;
-    }
-#if DEBUG_FS
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "On-demand encoded block for query `%s'\n",
-                 GNUNET_h2s (key));
-#endif  
-  cont (cont_cls,
-       key,
-       nsize,
-       edata,
-       GNUNET_BLOCK_TYPE_DBLOCK,
-       priority,
-       anonymity,
-       expiration,
-       uid);
+  GNUNET_CRYPTO_symmetric_encrypt (ndata, nsize, &skey, &iv, edata);
+  GNUNET_CRYPTO_hash (edata, nsize, &query);
+  if (0 != memcmp (&query, key, sizeof (struct GNUNET_HashCode)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("Indexed file `%s' changed at offset %llu\n"), fn,
+                (unsigned long long) off);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1,
+                             GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "On-demand encoded block for query `%s'\n", GNUNET_h2s (key));
+  cont (cont_cls, key, nsize, edata, GNUNET_BLOCK_TYPE_FS_DBLOCK, priority,
+        anonymity, expiration, uid);

   return GNUNET_OK;
 }
 
 
 /**
   return GNUNET_OK;
 }
 
 
 /**

- * Task run during shutdown.
- *
- * @param cls unused
- * @param tc unused
+ * Shutdown the module.

  */
  */

-static void
-shutdown_task (void *cls,
-              const struct GNUNET_SCHEDULER_TaskContext *tc)
+void
+GNUNET_FS_indexing_done ()

 {
 {

-  struct IndexInfo *pos;  
+  struct IndexInfo *pos;

 
 

+  while (NULL != (pos = indexed_files_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (indexed_files_head,
+                                indexed_files_tail,
+                                pos);
+    if (pos->fhc != NULL)
+      GNUNET_CRYPTO_hash_file_cancel (pos->fhc);
+    GNUNET_break (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_remove (ifm,
+                                                       &pos->file_id, pos));
+    GNUNET_free (pos);
+  }

   GNUNET_CONTAINER_multihashmap_destroy (ifm);
   ifm = NULL;
   GNUNET_CONTAINER_multihashmap_destroy (ifm);
   ifm = NULL;

-  while (NULL != (pos = indexed_files))
-    {
-      indexed_files = pos->next;
-      GNUNET_free (pos);
-    }
-  sched = NULL;

   cfg = NULL;
 }
 
   cfg = NULL;
 }
 


@@ -690,20 +614,16 @@ shutdown_task (void *cls,
 /**
  * Initialize the indexing submodule.
  *
 /**
  * Initialize the indexing submodule.
  *

- * @param s scheduler to use

  * @param c configuration to use
  * @param c configuration to use

+ * @param d datastore to use

  */
 int
  */
 int

-GNUNET_FS_indexing_init (struct GNUNET_SCHEDULER_Handle *s,
-                        const struct GNUNET_CONFIGURATION_Handle *c)
+GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c,
+                         struct GNUNET_DATASTORE_Handle *d)

 {
 {

-  sched = s;

   cfg = c;
   cfg = c;

-  ifm = GNUNET_CONTAINER_multihashmap_create (128);
-  GNUNET_SCHEDULER_add_delayed (sched,
-                               GNUNET_TIME_UNIT_FOREVER_REL,
-                               &shutdown_task,
-                               NULL);
+  dsh = d;
+  ifm = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_YES);

   read_index_list ();
   return GNUNET_OK;
 }
   read_index_list ();
   return GNUNET_OK;
 }