fix #3704
[oweals/gnunet.git] / src / fs / fs_unindex.c
index ef63e798ebbd9c0b44ed473deb0393cf904ec381..c87c4e4e9ae342ccaf06fab28fb2051b636eb4e3 100644 (file)
@@ -1,10 +1,10 @@
 /*
      This file is part of GNUnet.
 /*
      This file is part of GNUnet.
-     (C) 2003, 2004, 2006 Christian Grothoff (and other contributing authors)
+     Copyright (C) 2003--2013 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
 */
 
 /**
 */
 
 /**
- * @file applications/fs/ecrs/unindex.c
- * @author Krista Bennett
+ * @file fs/fs_unindex.c
+ * @author Krista Grothoff
  * @author Christian Grothoff
  * @author Christian Grothoff
+ * @brief Unindex file.
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_fs_service.h"
+#include "gnunet_protocols.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+#include "block_fs.h"
+#include "fs_publish_ublock.h"
+
+
+/**
+ * Function called by the tree encoder to obtain
+ * a block of plaintext data (for the lowest level
+ * of the tree).
  *
  *
- * Unindex file.
+ * @param cls our publishing context
+ * @param offset identifies which block to get
+ * @param max (maximum) number of bytes to get; returning
+ *        fewer will also cause errors
+ * @param buf where to copy the plaintext buffer
+ * @param emsg location to store an error message (on error)
+ * @return number of bytes copied to buf, 0 on error
+ */
+static size_t
+unindex_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  size_t pt_size;
+
+  pt_size = GNUNET_MIN (max, uc->file_size - offset);
+  if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET))
+  {
+    *emsg = GNUNET_strdup (_("Failed to find given position in file"));
+    return 0;
+  }
+  if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size))
+  {
+    *emsg = GNUNET_strdup (_("Failed to read file"));
+    return 0;
+  }
+  return pt_size;
+}
+
+
+/**
+ * Fill in all of the generic fields for
+ * an unindex event and call the callback.
  *
  *
- * TODO:
- * - code cleanup (share more with upload.c)
+ * @param pi structure to fill in
+ * @param uc overall unindex context
+ * @param offset where we are in the file (for progress)
  */
  */
+void
+GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_UnindexContext *uc,
+                                uint64_t offset)
+{
+  pi->value.unindex.uc = uc;
+  pi->value.unindex.cctx = uc->client_info;
+  pi->value.unindex.filename = uc->filename;
+  pi->value.unindex.size = uc->file_size;
+  pi->value.unindex.eta =
+      GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size);
+  pi->value.unindex.duration =
+      GNUNET_TIME_absolute_get_duration (uc->start_time);
+  pi->value.unindex.completed = offset;
+  pi->fsh = uc->h;
+  uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi);
+}
 
 
-#include "platform.h"
-#include "gnunet_protocols.h"
-#include "gnunet_ecrs_lib.h"
-#include "gnunet_fs_lib.h"
-#include "gnunet_getoption_lib.h"
-#include "ecrs_core.h"
-#include "ecrs.h"
-#include "fs.h"
-#include "tree.h"
 
 
-#define STRICT_CHECKS GNUNET_NO
+/**
+ * Function called with information about our
+ * progress in computing the tree encoding.
+ *
+ * @param cls closure
+ * @param offset where are we in the file
+ * @param pt_block plaintext of the currently processed block
+ * @param pt_size size of pt_block
+ * @param depth depth of the block in the tree, 0 for DBLOCK
+ */
+static void
+unindex_progress (void *cls, uint64_t offset, const void *pt_block,
+                  size_t pt_size, unsigned int depth)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS;
+  pi.value.unindex.specifics.progress.data = pt_block;
+  pi.value.unindex.specifics.progress.offset = offset;
+  pi.value.unindex.specifics.progress.data_len = pt_size;
+  pi.value.unindex.specifics.progress.depth = depth;
+  GNUNET_FS_unindex_make_status_ (&pi, uc, offset);
+}
+
 
 /**
 
 /**
- * Append the given key and query to the iblock[level].
- * If iblock[level] is already full, compute its chk
- * and push it to level+1.  iblocks is guaranteed to
- * be big enough.
+ * We've encountered an error during
+ * unindexing.  Signal the client.
  *
  *
- * This function matches exactly upload.c::pushBlock,
- * except in the call to 'GNUNET_FS_delete'.  TODO: refactor
- * to avoid code duplication (move to block.c, pass
- * GNUNET_FS_delete as argument!).
+ * @param uc context for the failed unindexing operation
  */
  */
-static int
-pushBlock (struct GNUNET_ClientServerConnection *sock,
-           const GNUNET_EC_ContentHashKey * chk, unsigned int level,
-           GNUNET_DatastoreValue ** iblocks)
+static void
+signal_unindex_error (struct GNUNET_FS_UnindexContext *uc)
 {
 {
-  unsigned int size;
-  unsigned int present;
-  GNUNET_DatastoreValue *value;
-  GNUNET_EC_DBlock *db;
-  GNUNET_EC_ContentHashKey ichk;
-
-  size = ntohl (iblocks[level]->size) - sizeof (GNUNET_DatastoreValue);
-  present =
-    (size - sizeof (GNUNET_EC_DBlock)) / sizeof (GNUNET_EC_ContentHashKey);
-  db = (GNUNET_EC_DBlock *) & iblocks[level][1];
-  if (present == GNUNET_ECRS_CHK_PER_INODE)
-    {
-      GNUNET_EC_file_block_get_key (db, size, &ichk.key);
-      GNUNET_EC_file_block_get_query (db, size, &ichk.query);
-      if (GNUNET_OK != pushBlock (sock, &ichk, level + 1, iblocks))
-        {
-          GNUNET_GE_BREAK (NULL, 0);
-          return GNUNET_SYSERR;
-        }
-      GNUNET_EC_file_block_encode (db, size, &ichk.query, &value);
-#if STRICT_CHECKS
-      if (GNUNET_SYSERR == GNUNET_FS_delete (sock, value))
-        {
-          GNUNET_free (value);
-          GNUNET_GE_BREAK (NULL, 0);
-          return GNUNET_SYSERR;
-        }
-#else
-      GNUNET_FS_delete (sock, value);
-#endif
-      GNUNET_free (value);
-      size = sizeof (GNUNET_EC_DBlock);
-    }
-  /* append GNUNET_EC_ContentHashKey */
-  memcpy (&((char *) db)[size], chk, sizeof (GNUNET_EC_ContentHashKey));
-  iblocks[level]->size = htonl (size +
-                                sizeof (GNUNET_EC_ContentHashKey) +
-                                sizeof (GNUNET_DatastoreValue));
-  return GNUNET_OK;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+  pi.value.unindex.specifics.error.message = uc->emsg;
+  GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
 }
 
 
 }
 
 
+/**
+ * Continuation called to notify client about result of the
+ * datastore removal operation.
+ *
+ * @param cls closure
+ * @param success GNUNET_SYSERR on failure
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  if (success == GNUNET_SYSERR)
+  {
+    uc->emsg = GNUNET_strdup (msg);
+    signal_unindex_error (uc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Datastore REMOVE operation succeeded\n");
+  GNUNET_FS_tree_encoder_next (uc->tc);
+}
+
 
 /**
 
 /**
- * Undo sym-linking operation:
- * a) check if we have a symlink
- * b) delete symbolic link
+ * Function called asking for the current (encoded)
+ * block to be processed.  After processing the
+ * client should either call "GNUNET_FS_tree_encode_next"
+ * or (on error) "GNUNET_FS_tree_encode_finish".
+ *
+ * @param cls closure
+ * @param chk content hash key for the block (key for lookup in the datastore)
+ * @param offset offset of the block
+ * @param depth depth of the block, 0 for DBLOCK
+ * @param type type of the block (IBLOCK or DBLOCK)
+ * @param block the (encrypted) block
+ * @param block_size size of block (in bytes)
  */
  */
-static int
-undoSymlinking (struct GNUNET_GE_Context *ectx,
-                const char *fn,
-                const GNUNET_HashCode * fileId,
-                struct GNUNET_ClientServerConnection *sock)
+static void
+unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset,
+                 unsigned int depth, enum GNUNET_BLOCK_Type type,
+                 const void *block, uint16_t block_size)
 {
 {
-  GNUNET_EncName enc;
-  char *serverDir;
-  char *serverFN;
-  struct stat buf;
-
-#ifndef S_ISLNK
-  if (1)
-    return GNUNET_OK;           /* symlinks do not exist? */
-#endif
-  if (0 != LSTAT (fn, &buf))
-    {
-      GNUNET_GE_LOG_STRERROR_FILE (ectx,
-                                   GNUNET_GE_ERROR | GNUNET_GE_BULK |
-                                   GNUNET_GE_USER | GNUNET_GE_ADMIN, "stat",
-                                   fn);
-      return GNUNET_SYSERR;
-    }
-#ifdef S_ISLNK
-  if (!S_ISLNK (buf.st_mode))
-    return GNUNET_OK;
-#endif
-  serverDir =
-    GNUNET_get_daemon_configuration_value (sock, "FS", "INDEX-DIRECTORY");
-  if (serverDir == NULL)
-    return GNUNET_OK;
-  serverFN = GNUNET_malloc (strlen (serverDir) + 2 + sizeof (GNUNET_EncName));
-  strcpy (serverFN, serverDir);
-  GNUNET_free (serverDir);
-  if (serverFN[strlen (serverFN) - 1] != DIR_SEPARATOR)
-    strcat (serverFN, DIR_SEPARATOR_STR);
-  GNUNET_hash_to_enc (fileId, &enc);
-  strcat (serverFN, (char *) &enc);
-
-  if (0 != UNLINK (serverFN))
-    {
-      GNUNET_GE_LOG_STRERROR_FILE (ectx,
-                                   GNUNET_GE_ERROR | GNUNET_GE_BULK |
-                                   GNUNET_GE_USER | GNUNET_GE_ADMIN, "unlink",
-                                   serverFN);
-      GNUNET_free (serverFN);
-      return GNUNET_SYSERR;
-    }
-  GNUNET_free (serverFN);
-  return GNUNET_OK;
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  uint32_t size;
+  const void *data;
+  struct OnDemandBlock odb;
+
+  if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK)
+  {
+    size = block_size;
+    data = block;
+  }
+  else                          /* on-demand encoded DBLOCK */
+  {
+    size = sizeof (struct OnDemandBlock);
+    odb.offset = GNUNET_htonll (offset);
+    odb.file_id = uc->file_id;
+    data = &odb;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending REMOVE request to DATASTORE service\n");
+  GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1,
+                           GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc);
+  uc->chk = *chk;
 }
 
 
 }
 
 
+/**
+ * Function called with the response from the
+ * FS service to our unindexing request.
+ *
+ * @param cls closure, unindex context
+ * @param msg NULL on timeout, otherwise the response
+ */
+static void
+process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (uc->client != NULL)
+  {
+    GNUNET_CLIENT_disconnect (uc->client);
+    uc->client = NULL;
+  }
+  if (uc->state != UNINDEX_STATE_FS_NOTIFY)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg =
+        GNUNET_strdup (_("Unexpected time for a response from `fs' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  if (NULL == msg)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Timeout waiting for `fs' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Invalid response from `fs' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->state = UNINDEX_STATE_COMPLETE;
+  pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_make_status_ (&pi, uc, uc->file_size);
+}
+
 
 /**
 
 /**
- * Unindex a file.
+ * Function called when we are done with removing UBlocks.
+ * Disconnect from datastore and notify FS service about
+ * the unindex event.
  *
  *
- * @return GNUNET_SYSERR if the unindexing failed (i.e. not indexed)
+ * @param uc our unindexing context
  */
  */
-int
-GNUNET_ECRS_file_unindex (struct GNUNET_GE_Context *ectx,
-                          struct GNUNET_GC_Configuration *cfg,
-                          const char *filename,
-                          GNUNET_ECRS_UploadProgressCallback upcb,
-                          void *upcbClosure, GNUNET_ECRS_TestTerminate tt,
-                          void *ttClosure)
+static void
+unindex_finish (struct GNUNET_FS_UnindexContext *uc)
 {
 {
-  unsigned long long filesize;
-  unsigned long long pos;
-  unsigned int treedepth;
-  int fd;
-  int i;
-  unsigned int size;
-  GNUNET_DatastoreValue **iblocks;
-  GNUNET_DatastoreValue *dblock;
-  GNUNET_EC_DBlock *db;
-  GNUNET_DatastoreValue *value;
-  struct GNUNET_ClientServerConnection *sock;
-  GNUNET_HashCode fileId;
-  GNUNET_EC_ContentHashKey chk;
-  GNUNET_CronTime eta;
-  GNUNET_CronTime start;
-  GNUNET_CronTime now;
-  int wasIndexed;
-
-  start = GNUNET_get_time ();
-  if (GNUNET_YES != GNUNET_disk_file_test (ectx, filename))
-    {
-      GNUNET_GE_BREAK (ectx, 0);
-      return GNUNET_SYSERR;
-    }
-  if (GNUNET_OK !=
-      GNUNET_disk_file_size (ectx, filename, &filesize, GNUNET_YES))
-    return GNUNET_SYSERR;
-  sock = GNUNET_client_connection_create (ectx, cfg);
-  if (sock == NULL)
-    return GNUNET_SYSERR;
-  eta = 0;
-  if (upcb != NULL)
-    upcb (filesize, 0, eta, upcbClosure);
-  if (GNUNET_SYSERR == GNUNET_hash_file (ectx, filename, &fileId))
-    {
-      GNUNET_client_connection_destroy (sock);
-      GNUNET_GE_BREAK (ectx, 0);
-      return GNUNET_SYSERR;
-    }
-  now = GNUNET_get_time ();
-  eta = now + 2 * (now - start);
-  /* very rough estimate: GNUNET_hash reads once through the file,
-     we'll do that once more and write it.  But of course
-     the second read may be cached, and we have the encryption,
-     so a factor of two is really, really just a rough estimate */
-  start = now;
-  /* reset the counter since the formula later does not
-     take the time for GNUNET_hash_file into account */
-  treedepth = GNUNET_ECRS_compute_depth (filesize);
-
-  /* Test if file is indexed! */
-  wasIndexed = GNUNET_FS_test_indexed (sock, &fileId);
-
-  fd = GNUNET_disk_file_open (ectx, filename, O_RDONLY | O_LARGEFILE);
-  if (fd == -1)
+  char *emsg;
+  struct UnindexMessage req;
+
+  /* generate final progress message */
+  unindex_progress (uc, uc->file_size, NULL, 0, 0);
+  GNUNET_FS_tree_encoder_finish (uc->tc, &emsg);
+  uc->tc = NULL;
+  GNUNET_DISK_file_close (uc->fh);
+  uc->fh = NULL;
+  GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+  uc->dsh = NULL;
+  uc->state = UNINDEX_STATE_FS_NOTIFY;
+  GNUNET_FS_unindex_sync_ (uc);
+  uc->client = GNUNET_CLIENT_connect ("fs", uc->h->cfg);
+  if (uc->client == NULL)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg =
+        GNUNET_strdup (_("Failed to connect to FS service for unindexing."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending UNINDEX message to FS service\n");
+  req.header.size = htons (sizeof (struct UnindexMessage));
+  req.header.type = htons (GNUNET_MESSAGE_TYPE_FS_UNINDEX);
+  req.reserved = 0;
+  req.file_id = uc->file_id;
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CLIENT_transmit_and_get_response (uc->client,
+                                                         &req.header,
+                                                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                                                         GNUNET_YES,
+                                                         &process_fs_response,
+                                                         uc));
+}
+
+
+
+/**
+ * Function called by the directory scanner as we extract keywords
+ * that we will need to remove UBlocks.
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param filename which file we are making progress on
+ * @param is_directory GNUNET_YES if this is a directory,
+ *                     GNUNET_NO if this is a file
+ *                     GNUNET_SYSERR if it is neither (or unknown)
+ * @param reason kind of progress we are making
+ */
+static void
+unindex_directory_scan_cb (void *cls,
+                          const char *filename,
+                          int is_directory,
+                          enum GNUNET_FS_DirScannerProgressUpdateReason reason)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  static struct GNUNET_FS_ShareTreeItem * directory_scan_result;
+
+  switch (reason)
+  {
+  case GNUNET_FS_DIRSCANNER_FINISHED:
+    directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan);
+    uc->dscan = NULL;
+    if (NULL != directory_scan_result->ksk_uri)
     {
     {
-      GNUNET_client_connection_destroy (sock);
-      return GNUNET_SYSERR;
+      uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri);
+      uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS;
+      GNUNET_FS_unindex_sync_ (uc);
+      GNUNET_FS_unindex_do_remove_kblocks_ (uc);
     }
     }
-  dblock =
-    GNUNET_malloc (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
-                   sizeof (GNUNET_EC_DBlock));
-  dblock->size =
-    htonl (sizeof (GNUNET_DatastoreValue) + GNUNET_ECRS_DBLOCK_SIZE +
-           sizeof (GNUNET_EC_DBlock));
-  dblock->anonymity_level = htonl (0);
-  dblock->priority = htonl (0);
-  dblock->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-  dblock->expiration_time = GNUNET_htonll (0);
-  db = (GNUNET_EC_DBlock *) & dblock[1];
-  db->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-  iblocks =
-    GNUNET_malloc (sizeof (GNUNET_DatastoreValue *) * (treedepth + 1));
-  for (i = 0; i <= treedepth; i++)
+    else
     {
     {
-      iblocks[i] =
-        GNUNET_malloc (sizeof (GNUNET_DatastoreValue) +
-                       GNUNET_ECRS_IBLOCK_SIZE + sizeof (GNUNET_EC_DBlock));
-      iblocks[i]->size =
-        htonl (sizeof (GNUNET_DatastoreValue) + sizeof (GNUNET_EC_DBlock));
-      iblocks[i]->anonymity_level = htonl (0);
-      iblocks[i]->priority = htonl (0);
-      iblocks[i]->type = htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
-      iblocks[i]->expiration_time = GNUNET_htonll (0);
-      ((GNUNET_EC_DBlock *) & iblocks[i][1])->type =
-        htonl (GNUNET_ECRS_BLOCKTYPE_DATA);
+      uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan."));
+      GNUNET_FS_unindex_sync_ (uc);
+      unindex_finish (uc);
     }
     }
+    GNUNET_FS_share_tree_free (directory_scan_result);
+    break;
+  case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+               _("Internal error scanning `%s'.\n"),
+               uc->filename);
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+    uc->emsg = GNUNET_strdup (_("Failed to get KSKs from directory scan."));
+    GNUNET_FS_unindex_sync_ (uc);
+    unindex_finish (uc);
+    break;
+  default:
+    break;
+  }
+}
+
+
+/**
+ * If necessary, connect to the datastore and remove the UBlocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  char *ex;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS", &ex))
+    ex = NULL;
+  uc->dscan = GNUNET_FS_directory_scan_start (uc->filename,
+                                             GNUNET_NO, ex,
+                                             &unindex_directory_scan_cb,
+                                             uc);
+  GNUNET_free_non_null (ex);
+}
 
 
-  pos = 0;
-  while (pos < filesize)
+
+/**
+ * Continuation called to notify client about result of the remove
+ * operation for the UBlock.
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
+ *                GNUNET_NO if content was already there
+ *                GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority content to be stored
+ *                by the datacache at this time, zero for unknown, forever if we have no
+ *                space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+continue_after_remove (void *cls,
+                      int32_t success,
+                      struct GNUNET_TIME_Absolute min_expiration,
+                      const char *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->dqe = NULL;
+  if (success != GNUNET_YES)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+               _("Failed to remove UBlock: %s\n"),
+               msg);
+  uc->ksk_offset++;
+  GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+}
+
+
+/**
+ * Function called from datastore with result from us looking for
+ * a UBlock.  There are four cases:
+ * 1) no result, means we move on to the next keyword
+ * 2) UID is the same as the first UID, means we move on to next keyword
+ * 3) UBlock for a different CHK, means we keep looking for more
+ * 4) UBlock is for our CHK, means we remove the block and then move
+ *           on to the next keyword
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+process_kblock_for_unindex (void *cls,
+                           const struct GNUNET_HashCode *key,
+                           size_t size, const void *data,
+                           enum GNUNET_BLOCK_Type type,
+                           uint32_t priority,
+                           uint32_t anonymity,
+                           struct GNUNET_TIME_Absolute
+                           expiration, uint64_t uid)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  const struct UBlock *ub;
+  struct GNUNET_FS_Uri *chk_uri;
+  struct GNUNET_HashCode query;
+
+  uc->dqe = NULL;
+  if (NULL == data)
+  {
+    /* no result */
+    uc->ksk_offset++;
+    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+    return;
+  }
+  if (0 == uc->first_uid)
+  {
+    /* remember UID of first result to detect cycles */
+    uc->first_uid = uid;
+  }
+  else if (uid == uc->first_uid)
+  {
+    /* no more additional results */
+    uc->ksk_offset++;
+    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+    return;
+  }
+  GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
+  if (size < sizeof (struct UBlock))
+  {
+    GNUNET_break (0);
+    goto get_next;
+  }
+  ub = data;
+  GNUNET_CRYPTO_hash (&ub->verification_key,
+                     sizeof (ub->verification_key),
+                     &query);
+  if (0 != memcmp (&query, key, sizeof (struct GNUNET_HashCode)))
+  {
+    /* result does not match our keyword, skip */
+    goto get_next;
+  }
+  {
+    char pt[size - sizeof (struct UBlock)];
+    struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+    const char *keyword;
+
+    GNUNET_CRYPTO_ecdsa_key_get_public (GNUNET_CRYPTO_ecdsa_key_get_anonymous (),
+                                                   &anon_pub);
+    keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
+    GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof (struct UBlock),
+                              &anon_pub,
+                              keyword,
+                              pt);
+    if (NULL == memchr (&pt[1], 0, sizeof (pt) - 1))
     {
     {
-      if (upcb != NULL)
-        upcb (filesize, pos, eta, upcbClosure);
-      if (tt != NULL)
-        if (GNUNET_OK != tt (ttClosure))
-          goto FAILURE;
-      size = GNUNET_ECRS_DBLOCK_SIZE;
-      if (size > filesize - pos)
-        {
-          size = filesize - pos;
-          memset (&db[1], 0, GNUNET_ECRS_DBLOCK_SIZE);
-        }
-      dblock->size =
-        htonl (sizeof (GNUNET_DatastoreValue) + size +
-               sizeof (GNUNET_EC_DBlock));
-      if (size != READ (fd, &db[1], size))
-        {
-          GNUNET_GE_LOG_STRERROR_FILE (ectx,
-                                       GNUNET_GE_ERROR | GNUNET_GE_USER |
-                                       GNUNET_GE_ADMIN | GNUNET_GE_BULK,
-                                       "READ", filename);
-          goto FAILURE;
-        }
-      if (tt != NULL)
-        if (GNUNET_OK != tt (ttClosure))
-          goto FAILURE;
-      GNUNET_EC_file_block_get_key (db, size + sizeof (GNUNET_EC_DBlock),
-                                    &chk.key);
-      GNUNET_EC_file_block_get_query (db, size + sizeof (GNUNET_EC_DBlock),
-                                      &chk.query);
-      if (GNUNET_OK != pushBlock (sock, &chk, 0,        /* dblocks are on level 0 */
-                                  iblocks))
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-      if (!wasIndexed)
-        {
-          if (GNUNET_OK ==
-              GNUNET_EC_file_block_encode (db, size, &chk.query, &value))
-            {
-              *value = *dblock; /* copy options! */
-#if STRICT_CHECKS
-              if (GNUNET_OK != GNUNET_FS_delete (sock, value))
-                {
-                  GNUNET_free (value);
-                  GNUNET_GE_BREAK (ectx, 0);
-                  goto FAILURE;
-                }
-#else
-              GNUNET_FS_delete (sock, value);
-#endif
-              GNUNET_free (value);
-            }
-          else
-            {
-              goto FAILURE;
-            }
-        }
-      pos += size;
-      now = GNUNET_get_time ();
-      eta = (GNUNET_CronTime) (start +
-                               (((double) (now - start) / (double) pos))
-                               * (double) filesize);
+      GNUNET_break_op (0); /* malformed UBlock */
+      goto get_next;
     }
     }
-  if (tt != NULL)
-    if (GNUNET_OK != tt (ttClosure))
-      goto FAILURE;
-  for (i = 0; i < treedepth; i++)
+    chk_uri = GNUNET_FS_uri_parse (&pt[1], NULL);
+    if (NULL == chk_uri)
     {
     {
-      size = ntohl (iblocks[i]->size) - sizeof (GNUNET_DatastoreValue);
-      db = (GNUNET_EC_DBlock *) & iblocks[i][1];
-      GNUNET_EC_file_block_get_key (db, size, &chk.key);
-      GNUNET_EC_file_block_get_query (db, size, &chk.query);
-      if (GNUNET_OK != pushBlock (sock, &chk, i + 1, iblocks))
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-      GNUNET_EC_file_block_encode (db, size, &chk.query, &value);
-#if STRICT_CHECKS
-      if (GNUNET_OK != GNUNET_FS_delete (sock, value))
-        {
-          GNUNET_free (value);
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-#else
-      GNUNET_FS_delete (sock, value);
-#endif
-      GNUNET_free (value);
-      GNUNET_free (iblocks[i]);
-      iblocks[i] = NULL;
+      GNUNET_break_op (0); /* malformed UBlock */
+      goto get_next;
     }
     }
+  }
+  if (0 != memcmp (&uc->chk,
+                  &chk_uri->data.chk.chk,
+                  sizeof (struct ContentHashKey)))
+  {
+    /* different CHK, ignore */
+    GNUNET_FS_uri_destroy (chk_uri);
+    goto get_next;
+  }
+  GNUNET_FS_uri_destroy (chk_uri);
+  /* matches! */
+  uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
+                                    key, size, data,
+                                    0 /* priority */, 1 /* queue size */,
+                                    GNUNET_TIME_UNIT_FOREVER_REL,
+                                    &continue_after_remove,
+                                    uc);
+  return;
+ get_next:
+  uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
+                                     uc->roff++,
+                                     &uc->uquery,
+                                     GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                     0 /* priority */, 1 /* queue size */,
+                                     GNUNET_TIME_UNIT_FOREVER_REL,
+                                     &process_kblock_for_unindex,
+                                     uc);
+}
 
 
-  if (wasIndexed)
-    {
-      if (GNUNET_OK == undoSymlinking (ectx, filename, &fileId, sock))
-        {
-          if (GNUNET_OK !=
-              GNUNET_FS_unindex (sock, GNUNET_ECRS_DBLOCK_SIZE, &fileId))
-            {
-              GNUNET_GE_BREAK (ectx, 0);
-              goto FAILURE;
-            }
-        }
-      else
-        {
-          GNUNET_GE_BREAK (ectx, 0);
-          goto FAILURE;
-        }
-    }
-  GNUNET_free (iblocks[treedepth]);
-  /* free resources */
-  GNUNET_free (iblocks);
-  GNUNET_free (dblock);
-  CLOSE (fd);
-  GNUNET_client_connection_destroy (sock);
-  return GNUNET_OK;
-FAILURE:
-  for (i = 0; i <= treedepth; i++)
-    GNUNET_free_non_null (iblocks[i]);
-  GNUNET_free (iblocks);
-  GNUNET_free (dblock);
-  CLOSE (fd);
-  GNUNET_client_connection_destroy (sock);
-  return GNUNET_SYSERR;
+
+/**
+ * If necessary, connect to the datastore and remove the KBlocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  const char *keyword;
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
+  struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+  struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
+
+  if (NULL == uc->dsh)
+    uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
+  if (NULL == uc->dsh)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  if ( (NULL == uc->ksk_uri) ||
+       (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount) )
+  {
+    unindex_finish (uc);
+    return;
+  }
+  anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub);
+  keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
+  GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub,
+                                      keyword,
+                                      "fs-ublock",
+                                      &dpub);
+  GNUNET_CRYPTO_hash (&dpub,
+                     sizeof (dpub),
+                     &uc->uquery);
+  uc->first_uid = 0;
+  uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
+                                     uc->roff++,
+                                     &uc->uquery,
+                                     GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                     0 /* priority */, 1 /* queue size */,
+                                     GNUNET_TIME_UNIT_FOREVER_REL,
+                                     &process_kblock_for_unindex,
+                                     uc);
+}
+
+
+/**
+ * Function called when the tree encoder has
+ * processed all blocks.  Clean up.
+ *
+ * @param cls our unindexing context
+ * @param tc not used
+ */
+static void
+unindex_extract_keywords (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_do_extract_keywords_ (uc);
+}
+
+
+/**
+ * Connect to the datastore and remove the blocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  if (NULL == uc->dsh)
+    uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
+  if (NULL == uc->dsh)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->fh =
+      GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ,
+                             GNUNET_DISK_PERM_NONE);
+  if (NULL == uc->fh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Failed to open file for unindexing."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->tc =
+      GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader,
+                                     &unindex_process, &unindex_progress,
+                                     &unindex_extract_keywords);
+  GNUNET_FS_tree_encoder_next (uc->tc);
+}
+
+
+/**
+ * Function called once the hash of the file
+ * that is being unindexed has been computed.
+ *
+ * @param cls closure, unindex context
+ * @param file_id computed hash, NULL on error
+ */
+void
+GNUNET_FS_unindex_process_hash_ (void *cls, const struct GNUNET_HashCode * file_id)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->fhc = NULL;
+  if (uc->state != UNINDEX_STATE_HASHING)
+  {
+    GNUNET_FS_unindex_stop (uc);
+    return;
+  }
+  if (file_id == NULL)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_("Failed to compute hash of file."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->file_id = *file_id;
+  uc->state = UNINDEX_STATE_DS_REMOVE;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_do_remove_ (uc);
+}
+
+
+/**
+ * Create SUSPEND event for the given unindex operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_UnindexContext` to signal for
+ */
+void
+GNUNET_FS_unindex_signal_suspend_ (void *cls)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  /* FIXME: lots of duplication with unindex_stop here! */
+  if (uc->dscan != NULL)
+  {
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+  }
+  if (NULL != uc->dqe)
+  {
+    GNUNET_DATASTORE_cancel (uc->dqe);
+    uc->dqe = NULL;
+  }
+  if (uc->fhc != NULL)
+  {
+    GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
+    uc->fhc = NULL;
+  }
+  if (NULL != uc->ksk_uri)
+  {
+    GNUNET_FS_uri_destroy (uc->ksk_uri);
+    uc->ksk_uri = NULL;
+  }
+  if (uc->client != NULL)
+  {
+    GNUNET_CLIENT_disconnect (uc->client);
+    uc->client = NULL;
+  }
+  if (NULL != uc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+  }
+  if (NULL != uc->tc)
+  {
+    GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
+    uc->tc = NULL;
+  }
+  if (uc->fh != NULL)
+  {
+    GNUNET_DISK_file_close (uc->fh);
+    uc->fh = NULL;
+  }
+  GNUNET_FS_end_top (uc->h, uc->top);
+  pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND;
+  GNUNET_FS_unindex_make_status_ (&pi, uc,
+                                  (uc->state ==
+                                   UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
+  GNUNET_break (NULL == uc->client_info);
+  GNUNET_free (uc->filename);
+  GNUNET_free_non_null (uc->serialization);
+  GNUNET_free_non_null (uc->emsg);
+  GNUNET_free (uc);
+}
+
+
+/**
+ * Unindex a file.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param filename file to unindex
+ * @param cctx initial value for the client context
+ * @return NULL on error, otherwise handle
+ */
+struct GNUNET_FS_UnindexContext *
+GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
+                        const char *filename,
+                         void *cctx)
+{
+  struct GNUNET_FS_UnindexContext *ret;
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t size;
+
+  if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES, GNUNET_YES))
+    return NULL;
+  ret = GNUNET_new (struct GNUNET_FS_UnindexContext);
+  ret->h = h;
+  ret->filename = GNUNET_strdup (filename);
+  ret->start_time = GNUNET_TIME_absolute_get ();
+  ret->file_size = size;
+  ret->client_info = cctx;
+  GNUNET_FS_unindex_sync_ (ret);
+  pi.status = GNUNET_FS_STATUS_UNINDEX_START;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+  GNUNET_FS_unindex_make_status_ (&pi, ret, 0);
+  ret->fhc =
+      GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, filename,
+                               HASHING_BLOCKSIZE,
+                               &GNUNET_FS_unindex_process_hash_, ret);
+  ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, ret);
+  return ret;
+}
+
+
+/**
+ * Clean up after completion of an unindex operation.
+ *
+ * @param uc handle
+ */
+void
+GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (uc->dscan != NULL)
+  {
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+  }
+  if (NULL != uc->dqe)
+  {
+    GNUNET_DATASTORE_cancel (uc->dqe);
+    uc->dqe = NULL;
+  }
+  if (uc->fhc != NULL)
+  {
+    GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
+    uc->fhc = NULL;
+  }
+  if (uc->client != NULL)
+  {
+    GNUNET_CLIENT_disconnect (uc->client);
+    uc->client = NULL;
+  }
+  if (NULL != uc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+  }
+  if (NULL != uc->ksk_uri)
+  {
+    GNUNET_FS_uri_destroy (uc->ksk_uri);
+    uc->ksk_uri = NULL;
+  }
+  if (NULL != uc->tc)
+  {
+    GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
+    uc->tc = NULL;
+  }
+  if (uc->fh != NULL)
+  {
+    GNUNET_DISK_file_close (uc->fh);
+    uc->fh = NULL;
+  }
+  GNUNET_FS_end_top (uc->h, uc->top);
+  if (uc->serialization != NULL)
+  {
+    GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                                 uc->serialization);
+    GNUNET_free (uc->serialization);
+    uc->serialization = NULL;
+  }
+  pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
+  GNUNET_FS_unindex_make_status_ (&pi, uc,
+                                  (uc->state ==
+                                   UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
+  GNUNET_break (NULL == uc->client_info);
+  GNUNET_free_non_null (uc->emsg);
+  GNUNET_free (uc->filename);
+  GNUNET_free (uc);
 }
 
 }
 
-/* end of unindex.c */
+/* end of fs_unindex.c */