X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Ffs%2Ffs_dirmetascan.c;h=45d502c91859552f33206d4cedbee218338cfb45;hb=dd1927b960c7cea13733e061a11142274652ba27;hp=6d97f15536fc734e16f8dcbfa4bdde2aee1a9f78;hpb=6f912718b1e65181e9e3a62c6c7b8cf32f7d1165;p=oweals%2Fgnunet.git diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c index 6d97f1553..45d502c91 100644 --- a/src/fs/fs_dirmetascan.c +++ b/src/fs/fs_dirmetascan.c @@ -1,10 +1,10 @@ /* This file is part of GNUnet - (C) 2005-2012 Christian Grothoff (and other contributing authors) + Copyright (C) 2005-2012 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your + by the Free Software Foundation; either version 3, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but @@ -14,8 +14,8 @@ You should have received a copy of the GNU General Public License along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. */ /** @@ -53,13 +53,13 @@ struct GNUNET_FS_DirScanner * Second argument to helper process. */ char *ex_arg; - + /** * The function that will be called every time there's a progress * message. */ GNUNET_FS_DirScannerProgressCallback progress_callback; - + /** * A closure for progress_callback. */ @@ -68,17 +68,31 @@ struct GNUNET_FS_DirScanner /** * After the scan is finished, it will contain a pointer to the * top-level directory entry in the directory tree built by the - * scanner. Must only be manipulated by the thread for the - * duration of the thread's runtime. + * scanner. */ struct GNUNET_FS_ShareTreeItem *toplevel; -}; + /** + * Current position during processing. + */ + struct GNUNET_FS_ShareTreeItem *pos; + + /** + * Task scheduled when we are done. + */ + struct GNUNET_SCHEDULER_Task * stop_task; + /** + * Arguments for helper. + */ + char *args[4]; + +}; /** - * Abort the scan. + * Abort the scan. Must not be called from within the progress_callback + * function. * * @param ds directory scanner structure */ @@ -86,12 +100,15 @@ void GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds) { /* terminate helper */ - GNUNET_HELPER_stop (ds->helper); + if (NULL != ds->helper) + GNUNET_HELPER_stop (ds->helper, GNUNET_NO); /* free resources */ if (NULL != ds->toplevel) GNUNET_FS_share_tree_free (ds->toplevel); - GNUNET_free (ds->ex_arg); + if (NULL != ds->stop_task) + GNUNET_SCHEDULER_cancel (ds->stop_task); + GNUNET_free_non_null (ds->ex_arg); GNUNET_free (ds->filename_expanded); GNUNET_free (ds); } @@ -114,12 +131,117 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds) GNUNET_assert (NULL == ds->helper); /* preserve result */ result = ds->toplevel; - ds->toplevel = NULL; + ds->toplevel = NULL; GNUNET_FS_directory_scan_abort (ds); return result; } +/** + * Move in the directory from the given position to the next file + * in DFS traversal. + * + * @param pos current position + * @return next file, NULL for none + */ +static struct GNUNET_FS_ShareTreeItem * +advance (struct GNUNET_FS_ShareTreeItem *pos) +{ + int moved; + + GNUNET_assert (NULL != pos); + moved = 0; /* must not terminate, even on file, otherwise "normal" */ + while ( (pos->is_directory == GNUNET_YES) || + (0 == moved) ) + { + if ( (moved != -1) && + (NULL != pos->children_head) ) + { + pos = pos->children_head; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->next) + { + pos = pos->next; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->parent) + { + pos = pos->parent; + moved = -1; /* force move to 'next' or 'parent' */ + continue; + } + /* no more options, end of traversal */ + return NULL; + } + return pos; +} + + +/** + * Add another child node to the tree. + * + * @param parent parent of the child, NULL for top level + * @param filename name of the file or directory + * @param is_directory GNUNET_YES for directories + * @return new entry that was just created + */ +static struct GNUNET_FS_ShareTreeItem * +expand_tree (struct GNUNET_FS_ShareTreeItem *parent, + const char *filename, + int is_directory) +{ + struct GNUNET_FS_ShareTreeItem *chld; + size_t slen; + + chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem); + chld->parent = parent; + chld->filename = GNUNET_strdup (filename); + GNUNET_asprintf (&chld->short_filename, + "%s%s", + GNUNET_STRINGS_get_short_name (filename), + is_directory == GNUNET_YES ? "/" : ""); + /* make sure we do not end with '//' */ + slen = strlen (chld->short_filename); + if ( (slen >= 2) && + (chld->short_filename[slen-1] == '/') && + (chld->short_filename[slen-2] == '/') ) + chld->short_filename[slen-1] = '\0'; + chld->is_directory = is_directory; + if (NULL != parent) + GNUNET_CONTAINER_DLL_insert (parent->children_head, + parent->children_tail, + chld); + return chld; +} + + +/** + * Task run last to shut everything down. + * + * @param cls the 'struct GNUNET_FS_DirScanner' + * @param tc unused + */ +static void +finish_scan (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DirScanner *ds = cls; + + ds->stop_task = NULL; + if (NULL != ds->helper) + { + GNUNET_HELPER_stop (ds->helper, GNUNET_NO); + ds->helper = NULL; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_FINISHED); +} + + /** * Called every time there is data to read from the scanner. * Calls the scanner progress handler. @@ -128,41 +250,188 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds) * @param client always NULL * @param msg message from the helper process */ -static void -process_helper_msgs (void *cls, +static int +process_helper_msgs (void *cls, void *client, const struct GNUNET_MessageHeader *msg) { struct GNUNET_FS_DirScanner *ds = cls; - ds++; + const char *filename; + size_t left; + #if 0 - enum GNUNET_FS_DirScannerProgressUpdateReason reason; - size_t filename_len; - int is_directory; - char *filename; - - /* Process message. If message is malformed or can't be read, end the scanner */ - /* read successfully, notify client about progress */ - ds->progress_callback (ds->progress_callback_cls, - ds, - filename, - is_directory, - reason); - GNUNET_free (filename); - - - /* having full filenames is too dangerous; always make sure we clean them up */ - item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); - - GNUNET_CONTAINER_meta_data_delete (item->meta, - EXTRACTOR_METATYPE_FILENAME, - NULL, 0); - GNUNET_CONTAINER_meta_data_insert (item->meta, "", - EXTRACTOR_METATYPE_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - item->short_filename, - strlen (item->short_filename) + 1); + fprintf (stderr, "DMS parses %u-byte message of type %u\n", + (unsigned int) ntohs (msg->size), + (unsigned int) ntohs (msg->type)); #endif + left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader); + filename = (const char*) &msg[1]; + switch (ntohs (msg->type)) + { + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_NO, + GNUNET_FS_DIRSCANNER_FILE_START); + if (NULL == ds->toplevel) + ds->toplevel = expand_tree (ds->pos, + filename, GNUNET_NO); + else + (void) expand_tree (ds->pos, + filename, GNUNET_NO); + return GNUNET_OK; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + if (0 == strcmp ("..", filename)) + { + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + ds->pos = ds->pos->parent; + return GNUNET_OK; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_FILE_START); + ds->pos = expand_tree (ds->pos, + filename, GNUNET_YES); + if (NULL == ds->toplevel) + ds->toplevel = ds->pos; + return GNUNET_OK; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR: + break; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE: + if ('\0' != filename[left-1]) + break; + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_FILE_IGNORED); + return GNUNET_OK; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE: + if (0 != left) + { + GNUNET_break (0); + break; + } + if (NULL == ds->toplevel) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_ALL_COUNTED); + ds->pos = ds->toplevel; + if (GNUNET_YES == ds->pos->is_directory) + ds->pos = advance (ds->pos); + return GNUNET_OK; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: + { + size_t nlen; + const char *end; + + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + end = memchr (filename, 0, left); + if (NULL == end) + { + GNUNET_break (0); + break; + } + end++; + nlen = end - filename; + left -= nlen; + if (0 != strcmp (filename, + ds->pos->filename)) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED); + if (0 < left) + { + ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left); + if (NULL == ds->pos->meta) + { + GNUNET_break (0); + break; + } + /* having full filenames is too dangerous; always make sure we clean them up */ + GNUNET_CONTAINER_meta_data_delete (ds->pos->meta, + EXTRACTOR_METATYPE_FILENAME, + NULL, 0); + /* instead, put in our 'safer' original filename */ + GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "", + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + ds->pos->short_filename, + strlen (ds->pos->short_filename) + 1); + } + ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (ds->pos->meta); + ds->pos = advance (ds->pos); + return GNUNET_OK; + } + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED: + if (NULL != ds->pos) + { + GNUNET_break (0); + break; + } + if (0 != left) + { + GNUNET_break (0); + break; + } + if (NULL == ds->toplevel) + { + GNUNET_break (0); + break; + } + ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, + ds); + return GNUNET_OK; + default: + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); + return GNUNET_OK; +} + + +/** + * Function called if our helper process died. + * + * @param cls the 'struct GNUNET_FS_DirScanner' callback. + */ +static void +helper_died_cb (void *cls) +{ + struct GNUNET_FS_DirScanner *ds = cls; + + ds->helper = NULL; + if (NULL != ds->stop_task) + return; /* normal death, was finished */ + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); } @@ -170,7 +439,7 @@ process_helper_msgs (void *cls, * Start a directory scanner thread. * * @param filename name of the directory to scan - * @param GNUNET_YES to not to run libextractor on files (only build a tree) + * @param disable_extractor #GNUNET_YES to not to run libextractor on files (only build a tree) * @param ex if not NULL, must be a list of extra plugins for extractor * @param cb the callback to call when there are scanning progress messages * @param cb_cls closure for 'cb' @@ -179,13 +448,12 @@ process_helper_msgs (void *cls, struct GNUNET_FS_DirScanner * GNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, - GNUNET_FS_DirScannerProgressCallback cb, + GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls) { struct stat sbuf; char *filename_expanded; struct GNUNET_FS_DirScanner *ds; - char *args[4]; if (0 != STAT (filename, &sbuf)) return NULL; @@ -195,21 +463,25 @@ GNUNET_FS_directory_scan_start (const char *filename, GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting to scan directory `%s'\n", filename_expanded); - ds = GNUNET_malloc (sizeof (struct GNUNET_FS_DirScanner)); + ds = GNUNET_new (struct GNUNET_FS_DirScanner); ds->progress_callback = cb; ds->progress_callback_cls = cb_cls; ds->filename_expanded = filename_expanded; - ds->ex_arg = GNUNET_strdup ((disable_extractor) ? "-" : ex); - args[0] = "gnunet-helper-fs-publish"; - args[1] = ds->filename_expanded; - args[2] = ds->ex_arg; - args[3] = NULL; - ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish", - args, + if (disable_extractor) + ds->ex_arg = GNUNET_strdup ("-"); + else + ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL; + ds->args[0] = "gnunet-helper-fs-publish"; + ds->args[1] = ds->filename_expanded; + ds->args[2] = ds->ex_arg; + ds->args[3] = NULL; + ds->helper = GNUNET_HELPER_start (GNUNET_NO, + "gnunet-helper-fs-publish", + ds->args, &process_helper_msgs, - ds); + &helper_died_cb, ds); if (NULL == ds->helper) - { + { GNUNET_free (filename_expanded); GNUNET_free (ds); return NULL;