Forget to commit some files

[oweals/gnunet.git] / src / fs / fs_dirmetascan.c

diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c
index 6d97f15536fc734e16f8dcbfa4bdde2aee1a9f78..45d502c91859552f33206d4cedbee218338cfb45 100644 (file)
--- a/src/fs/fs_dirmetascan.c
+++ b/src/fs/fs_dirmetascan.c
@@ -1,10 +1,10 @@
 /*
      This file is part of GNUnet
 /*
      This file is part of GNUnet

-     (C) 2005-2012 Christian Grothoff (and other contributing authors)
+     Copyright (C) 2005-2012 Christian Grothoff (and other contributing authors)

 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published

-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your

      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but


@@ -14,8 +14,8 @@
 
      You should have received a copy of the GNU General Public License
      along with GNUnet; see the file COPYING.  If not, write to the
 
      You should have received a copy of the GNU General Public License
      along with GNUnet; see the file COPYING.  If not, write to the

-     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-     Boston, MA 02111-1307, USA.
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.

 */
 
 /**
 */
 
 /**


@@ -53,13 +53,13 @@ struct GNUNET_FS_DirScanner
    * Second argument to helper process.
    */
   char *ex_arg;
    * Second argument to helper process.
    */
   char *ex_arg;

-  
+

   /**
    * The function that will be called every time there's a progress
    * message.
    */
   GNUNET_FS_DirScannerProgressCallback progress_callback;
   /**
    * The function that will be called every time there's a progress
    * message.
    */
   GNUNET_FS_DirScannerProgressCallback progress_callback;

-  
+

   /**
    * A closure for progress_callback.
    */
   /**
    * A closure for progress_callback.
    */


@@ -68,17 +68,31 @@ struct GNUNET_FS_DirScanner
   /**
    * After the scan is finished, it will contain a pointer to the
    * top-level directory entry in the directory tree built by the
   /**
    * After the scan is finished, it will contain a pointer to the
    * top-level directory entry in the directory tree built by the

-   * scanner.  Must only be manipulated by the thread for the
-   * duration of the thread's runtime.
+   * scanner.

    */
   struct GNUNET_FS_ShareTreeItem *toplevel;
 
    */
   struct GNUNET_FS_ShareTreeItem *toplevel;
 

-};
+  /**
+   * Current position during processing.
+   */
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  /**
+   * Task scheduled when we are done.
+   */
+  struct GNUNET_SCHEDULER_Task * stop_task;

 
 

+  /**
+   * Arguments for helper.
+   */
+  char *args[4];
+
+};

 
 
 /**
 
 
 /**

- * Abort the scan.
+ * Abort the scan.  Must not be called from within the progress_callback
+ * function.

  *
  * @param ds directory scanner structure
  */
  *
  * @param ds directory scanner structure
  */


@@ -86,12 +100,15 @@ void
 GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 {
   /* terminate helper */
 GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 {
   /* terminate helper */

-  GNUNET_HELPER_stop (ds->helper);
+  if (NULL != ds->helper)
+    GNUNET_HELPER_stop (ds->helper, GNUNET_NO);

 
   /* free resources */
   if (NULL != ds->toplevel)
     GNUNET_FS_share_tree_free (ds->toplevel);
 
   /* free resources */
   if (NULL != ds->toplevel)
     GNUNET_FS_share_tree_free (ds->toplevel);

-  GNUNET_free (ds->ex_arg);
+  if (NULL != ds->stop_task)
+    GNUNET_SCHEDULER_cancel (ds->stop_task);
+  GNUNET_free_non_null (ds->ex_arg);

   GNUNET_free (ds->filename_expanded);
   GNUNET_free (ds);
 }
   GNUNET_free (ds->filename_expanded);
   GNUNET_free (ds);
 }


@@ -114,12 +131,117 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
   GNUNET_assert (NULL == ds->helper);
   /* preserve result */
   result = ds->toplevel;
   GNUNET_assert (NULL == ds->helper);
   /* preserve result */
   result = ds->toplevel;

-  ds->toplevel = NULL; 
+  ds->toplevel = NULL;

   GNUNET_FS_directory_scan_abort (ds);
   return result;
 }
 
 
   GNUNET_FS_directory_scan_abort (ds);
   return result;
 }
 
 

+/**
+ * Move in the directory from the given position to the next file
+ * in DFS traversal.
+ *
+ * @param pos current position
+ * @return next file, NULL for none
+ */
+static struct GNUNET_FS_ShareTreeItem *
+advance (struct GNUNET_FS_ShareTreeItem *pos)
+{
+  int moved;
+
+  GNUNET_assert (NULL != pos);
+  moved = 0; /* must not terminate, even on file, otherwise "normal" */
+  while ( (pos->is_directory == GNUNET_YES) ||
+         (0 == moved) )
+  {
+    if ( (moved != -1) &&
+        (NULL != pos->children_head) )
+    {
+      pos = pos->children_head;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->next)
+    {
+      pos = pos->next;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->parent)
+    {
+      pos = pos->parent;
+      moved = -1; /* force move to 'next' or 'parent' */
+      continue;
+    }
+    /* no more options, end of traversal */
+    return NULL;
+  }
+  return pos;
+}
+
+
+/**
+ * Add another child node to the tree.
+ *
+ * @param parent parent of the child, NULL for top level
+ * @param filename name of the file or directory
+ * @param is_directory GNUNET_YES for directories
+ * @return new entry that was just created
+ */
+static struct GNUNET_FS_ShareTreeItem *
+expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
+            const char *filename,
+            int is_directory)
+{
+  struct GNUNET_FS_ShareTreeItem *chld;
+  size_t slen;
+
+  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
+  chld->parent = parent;
+  chld->filename = GNUNET_strdup (filename);
+  GNUNET_asprintf (&chld->short_filename,
+                  "%s%s",
+                  GNUNET_STRINGS_get_short_name (filename),
+                  is_directory == GNUNET_YES ? "/" : "");
+  /* make sure we do not end with '//' */
+  slen = strlen (chld->short_filename);
+  if ( (slen >= 2) &&
+       (chld->short_filename[slen-1] == '/') &&
+       (chld->short_filename[slen-2] == '/') )
+    chld->short_filename[slen-1] = '\0';
+  chld->is_directory = is_directory;
+  if (NULL != parent)
+      GNUNET_CONTAINER_DLL_insert (parent->children_head,
+                                  parent->children_tail,
+                                  chld);
+  return chld;
+}
+
+
+/**
+ * Task run last to shut everything down.
+ *
+ * @param cls the 'struct GNUNET_FS_DirScanner'
+ * @param tc unused
+ */
+static void
+finish_scan (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_FS_DirScanner *ds = cls;
+
+  ds->stop_task = NULL;
+  if (NULL != ds->helper)
+  {
+    GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
+    ds->helper = NULL;
+  }
+  ds->progress_callback (ds->progress_callback_cls,
+                        NULL, GNUNET_SYSERR,
+                        GNUNET_FS_DIRSCANNER_FINISHED);
+}
+
+

 /**
  * Called every time there is data to read from the scanner.
  * Calls the scanner progress handler.
 /**
  * Called every time there is data to read from the scanner.
  * Calls the scanner progress handler.


@@ -128,41 +250,188 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
  * @param client always NULL
  * @param msg message from the helper process
  */
  * @param client always NULL
  * @param msg message from the helper process
  */

-static void
-process_helper_msgs (void *cls, 
+static int
+process_helper_msgs (void *cls,

                     void *client,
                     const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;
                     void *client,
                     const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;

-  ds++;
+  const char *filename;
+  size_t left;
+

 #if 0
 #if 0

-  enum GNUNET_FS_DirScannerProgressUpdateReason reason;
-  size_t filename_len;
-  int is_directory;
-  char *filename;
-
-  /* Process message. If message is malformed or can't be read, end the scanner */
-  /* read successfully, notify client about progress */
-  ds->progress_callback (ds->progress_callback_cls, 
-                        ds, 
-                        filename,
-                        is_directory, 
-                        reason);
-  GNUNET_free (filename);
-
-
-  /* having full filenames is too dangerous; always make sure we clean them up */
-  item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename));
-
-  GNUNET_CONTAINER_meta_data_delete (item->meta, 
-                                    EXTRACTOR_METATYPE_FILENAME,
-                                    NULL, 0);
-  GNUNET_CONTAINER_meta_data_insert (item->meta, "<libgnunetfs>",
-                                     EXTRACTOR_METATYPE_FILENAME,
-                                     EXTRACTOR_METAFORMAT_UTF8, "text/plain",
-                                     item->short_filename, 
-                                    strlen (item->short_filename) + 1);
+  fprintf (stderr, "DMS parses %u-byte message of type %u\n",
+          (unsigned int) ntohs (msg->size),
+          (unsigned int) ntohs (msg->type));

 #endif
 #endif

+  left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
+  filename = (const char*) &msg[1];
+  switch (ntohs (msg->type))
+  {
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls,
+                          filename, GNUNET_NO,
+                          GNUNET_FS_DIRSCANNER_FILE_START);
+    if (NULL == ds->toplevel)
+      ds->toplevel = expand_tree (ds->pos,
+                                 filename, GNUNET_NO);
+    else
+      (void) expand_tree (ds->pos,
+                         filename, GNUNET_NO);
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 == strcmp ("..", filename))
+    {
+      if (NULL == ds->pos)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      ds->pos = ds->pos->parent;
+      return GNUNET_OK;
+    }
+    ds->progress_callback (ds->progress_callback_cls,
+                          filename, GNUNET_YES,
+                          GNUNET_FS_DIRSCANNER_FILE_START);
+    ds->pos = expand_tree (ds->pos,
+                          filename, GNUNET_YES);
+    if (NULL == ds->toplevel)
+      ds->toplevel = ds->pos;
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
+    break;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
+    if ('\0' != filename[left-1])
+      break;
+    ds->progress_callback (ds->progress_callback_cls,
+                          filename, GNUNET_SYSERR,
+                          GNUNET_FS_DIRSCANNER_FILE_IGNORED);
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (NULL == ds->toplevel)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls,
+                          NULL, GNUNET_SYSERR,
+                          GNUNET_FS_DIRSCANNER_ALL_COUNTED);
+    ds->pos = ds->toplevel;
+    if (GNUNET_YES == ds->pos->is_directory)
+      ds->pos = advance (ds->pos);
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA:
+    {
+      size_t nlen;
+      const char *end;
+
+      if (NULL == ds->pos)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      end = memchr (filename, 0, left);
+      if (NULL == end)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      end++;
+      nlen = end - filename;
+      left -= nlen;
+      if (0 != strcmp (filename,
+                      ds->pos->filename))
+      {
+       GNUNET_break (0);
+       break;
+      }
+      ds->progress_callback (ds->progress_callback_cls,
+                            filename, GNUNET_YES,
+                            GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
+      if (0 < left)
+      {
+       ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left);
+       if (NULL == ds->pos->meta)
+       {
+         GNUNET_break (0);
+         break;
+       }
+       /* having full filenames is too dangerous; always make sure we clean them up */
+       GNUNET_CONTAINER_meta_data_delete (ds->pos->meta,
+                                          EXTRACTOR_METATYPE_FILENAME,
+                                          NULL, 0);
+       /* instead, put in our 'safer' original filename */
+       GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
+                                          EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+                                          EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                          ds->pos->short_filename,
+                                          strlen (ds->pos->short_filename) + 1);
+      }
+      ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (ds->pos->meta);
+      ds->pos = advance (ds->pos);
+      return GNUNET_OK;
+    }
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
+    if (NULL != ds->pos)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (NULL == ds->toplevel)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan,
+                                             ds);
+    return GNUNET_OK;
+  default:
+    GNUNET_break (0);
+    break;
+  }
+  ds->progress_callback (ds->progress_callback_cls,
+                        NULL, GNUNET_SYSERR,
+                        GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called if our helper process died.
+ *
+ * @param cls the 'struct GNUNET_FS_DirScanner' callback.
+ */
+static void
+helper_died_cb (void *cls)
+{
+  struct GNUNET_FS_DirScanner *ds = cls;
+
+  ds->helper = NULL;
+  if (NULL != ds->stop_task)
+    return; /* normal death, was finished */
+  ds->progress_callback (ds->progress_callback_cls,
+                        NULL, GNUNET_SYSERR,
+                        GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);

 }
 
 
 }
 
 


@@ -170,7 +439,7 @@ process_helper_msgs (void *cls,
  * Start a directory scanner thread.
  *
  * @param filename name of the directory to scan
  * Start a directory scanner thread.
  *
  * @param filename name of the directory to scan

- * @param GNUNET_YES to not to run libextractor on files (only build a tree)
+ * @param disable_extractor #GNUNET_YES to not to run libextractor on files (only build a tree)

  * @param ex if not NULL, must be a list of extra plugins for extractor
  * @param cb the callback to call when there are scanning progress messages
  * @param cb_cls closure for 'cb'
  * @param ex if not NULL, must be a list of extra plugins for extractor
  * @param cb the callback to call when there are scanning progress messages
  * @param cb_cls closure for 'cb'


@@ -179,13 +448,12 @@ process_helper_msgs (void *cls,
 struct GNUNET_FS_DirScanner *
 GNUNET_FS_directory_scan_start (const char *filename,
                                int disable_extractor, const char *ex,
 struct GNUNET_FS_DirScanner *
 GNUNET_FS_directory_scan_start (const char *filename,
                                int disable_extractor, const char *ex,

-                               GNUNET_FS_DirScannerProgressCallback cb, 
+                               GNUNET_FS_DirScannerProgressCallback cb,

                                void *cb_cls)
 {
   struct stat sbuf;
   char *filename_expanded;
   struct GNUNET_FS_DirScanner *ds;
                                void *cb_cls)
 {
   struct stat sbuf;
   char *filename_expanded;
   struct GNUNET_FS_DirScanner *ds;

-  char *args[4];

 
   if (0 != STAT (filename, &sbuf))
     return NULL;
 
   if (0 != STAT (filename, &sbuf))
     return NULL;


@@ -195,21 +463,25 @@ GNUNET_FS_directory_scan_start (const char *filename,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting to scan directory `%s'\n",
              filename_expanded);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting to scan directory `%s'\n",
              filename_expanded);

-  ds = GNUNET_malloc (sizeof (struct GNUNET_FS_DirScanner));
+  ds = GNUNET_new (struct GNUNET_FS_DirScanner);

   ds->progress_callback = cb;
   ds->progress_callback_cls = cb_cls;
   ds->filename_expanded = filename_expanded;
   ds->progress_callback = cb;
   ds->progress_callback_cls = cb_cls;
   ds->filename_expanded = filename_expanded;

-  ds->ex_arg = GNUNET_strdup ((disable_extractor) ? "-" : ex);
-  args[0] = "gnunet-helper-fs-publish";
-  args[1] = ds->filename_expanded;
-  args[2] = ds->ex_arg;
-  args[3] = NULL;
-  ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish",
-                                   args,
+  if (disable_extractor)
+    ds->ex_arg = GNUNET_strdup ("-");
+  else
+    ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
+  ds->args[0] = "gnunet-helper-fs-publish";
+  ds->args[1] = ds->filename_expanded;
+  ds->args[2] = ds->ex_arg;
+  ds->args[3] = NULL;
+  ds->helper = GNUNET_HELPER_start (GNUNET_NO,
+                                   "gnunet-helper-fs-publish",
+                                   ds->args,

                                    &process_helper_msgs,
                                    &process_helper_msgs,

-                                   ds);
+                                   &helper_died_cb, ds);

   if (NULL == ds->helper)
   if (NULL == ds->helper)

-  {
+    {

     GNUNET_free (filename_expanded);
     GNUNET_free (ds);
     return NULL;
     GNUNET_free (filename_expanded);
     GNUNET_free (ds);
     return NULL;