From ee4d754ebbcc11369dd7e78b50db733e3b921fc2 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sun, 29 Jan 2012 21:20:17 +0000 Subject: [PATCH] -towards external meta scanner process --- src/fs/fs_dirmetascan.c | 265 +++++++++++++++++++++++++----- src/fs/gnunet-helper-fs-publish.c | 14 +- src/fs/gnunet-publish.c | 85 ++++++---- 3 files changed, 292 insertions(+), 72 deletions(-) diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c index 6d97f1553..69c7770db 100644 --- a/src/fs/fs_dirmetascan.c +++ b/src/fs/fs_dirmetascan.c @@ -68,11 +68,20 @@ struct GNUNET_FS_DirScanner /** * After the scan is finished, it will contain a pointer to the * top-level directory entry in the directory tree built by the - * scanner. Must only be manipulated by the thread for the - * duration of the thread's runtime. + * scanner. */ struct GNUNET_FS_ShareTreeItem *toplevel; + /** + * Current position during processing. + */ + struct GNUNET_FS_ShareTreeItem *pos; + + /** + * Arguments for helper. + */ + char *args[4]; + }; @@ -120,6 +129,76 @@ GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds) } +/** + * Move in the directory from the given position to the next file + * in DFS traversal. + * + * @param pos current position + * @return next file, NULL for none + */ +static struct GNUNET_FS_ShareTreeItem * +advance (struct GNUNET_FS_ShareTreeItem *pos) +{ + int moved; + + GNUNET_assert (NULL != pos); + moved = 0; /* must not terminate, even on file, otherwise "normal" */ + while ( (pos->is_directory) || + (0 == moved) ) + { + if ( (moved != -1) && + (NULL != pos->children_head) ) + { + pos = pos->children_head; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->next) + { + pos = pos->next; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->parent) + { + pos = pos->parent; + moved = -1; /* force move to 'next' or 'parent' */ + continue; + } + /* no more options, end of traversal */ + return NULL; + } + return pos; +} + + +/** + * Add another child node to the tree. + * + * @param parent parent of the child, NULL for top level + * @param filename name of the file or directory + * @param is_directory GNUNET_YES for directories + * @return new entry that was just created + */ +static struct GNUNET_FS_ShareTreeItem * +expand_tree (struct GNUNET_FS_ShareTreeItem *parent, + const char *filename, + int is_directory) +{ + struct GNUNET_FS_ShareTreeItem *chld; + + chld = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem)); + chld->parent = parent; + chld->filename = GNUNET_strdup (filename); + chld->is_directory = is_directory; + if (NULL != parent) + GNUNET_CONTAINER_DLL_insert (parent->children_head, + parent->children_tail, + chld); + return chld; +} + + /** * Called every time there is data to read from the scanner. * Calls the scanner progress handler. @@ -134,35 +213,145 @@ process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg) { struct GNUNET_FS_DirScanner *ds = cls; - ds++; -#if 0 - enum GNUNET_FS_DirScannerProgressUpdateReason reason; - size_t filename_len; - int is_directory; - char *filename; - - /* Process message. If message is malformed or can't be read, end the scanner */ - /* read successfully, notify client about progress */ + const char *filename; + size_t left; + + left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader); + filename = (const char*) &msg[1]; + switch (ntohs (msg->type)) + { + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_NO, + GNUNET_FS_DIRSCANNER_FILE_START); + expand_tree (ds->pos, + filename, GNUNET_NO); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + if (0 == strcmp ("..", filename)) + { + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + ds->pos = ds->pos->parent; + return; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_FILE_START); + ds->pos = expand_tree (ds->pos, + filename, GNUNET_YES); + if (NULL == ds->toplevel) + ds->toplevel = ds->pos; + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR: + break; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE: + if (filename[left-1] != '\0') + break; + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_FILE_IGNORED); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE: + if (0 != left) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_ALL_COUNTED); + ds->pos = ds->toplevel; + if (ds->pos->is_directory) + ds->pos = advance (ds->pos); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: + { + size_t nlen; + const char *end; + + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + end = memchr (filename, 0, left); + if (NULL == end) + { + GNUNET_break (0); + break; + } + end++; + nlen = end - filename; + left -= nlen; + if (0 != strcmp (filename, + ds->pos->filename)) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED); + if (0 < left) + { + ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left); + if (NULL == ds->pos->meta) + { + GNUNET_break (0); + break; + } + /* having full filenames is too dangerous; always make sure we clean them up */ + ds->pos->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); + GNUNET_CONTAINER_meta_data_delete (ds->pos->meta, + EXTRACTOR_METATYPE_FILENAME, + NULL, 0); + GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "", + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + ds->pos->short_filename, + strlen (ds->pos->short_filename) + 1); + } + ds->pos = advance (ds->pos); + return; + } + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED: + if (NULL != ds->pos) + { + GNUNET_break (0); + break; + } + if (0 != left) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); + + return; + default: + GNUNET_break (0); + break; + } ds->progress_callback (ds->progress_callback_cls, - ds, - filename, - is_directory, - reason); - GNUNET_free (filename); - - - /* having full filenames is too dangerous; always make sure we clean them up */ - item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); - - GNUNET_CONTAINER_meta_data_delete (item->meta, - EXTRACTOR_METATYPE_FILENAME, - NULL, 0); - GNUNET_CONTAINER_meta_data_insert (item->meta, "", - EXTRACTOR_METATYPE_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - item->short_filename, - strlen (item->short_filename) + 1); -#endif + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); + } @@ -185,7 +374,6 @@ GNUNET_FS_directory_scan_start (const char *filename, struct stat sbuf; char *filename_expanded; struct GNUNET_FS_DirScanner *ds; - char *args[4]; if (0 != STAT (filename, &sbuf)) return NULL; @@ -199,13 +387,16 @@ GNUNET_FS_directory_scan_start (const char *filename, ds->progress_callback = cb; ds->progress_callback_cls = cb_cls; ds->filename_expanded = filename_expanded; - ds->ex_arg = GNUNET_strdup ((disable_extractor) ? "-" : ex); - args[0] = "gnunet-helper-fs-publish"; - args[1] = ds->filename_expanded; - args[2] = ds->ex_arg; - args[3] = NULL; + if (disable_extractor) + ds->ex_arg = GNUNET_strdup ("-"); + else + ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL; + ds->args[0] = "gnunet-helper-fs-publish"; + ds->args[1] = ds->filename_expanded; + ds->args[2] = ds->ex_arg; + ds->args[3] = NULL; ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish", - args, + ds->args, &process_helper_msgs, ds); if (NULL == ds->helper) diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c index 21ec19fe9..7accf79ac 100644 --- a/src/fs/gnunet-helper-fs-publish.c +++ b/src/fs/gnunet-helper-fs-publish.c @@ -223,9 +223,10 @@ scan_callback (void *cls, return GNUNET_SYSERR; } chld->parent = rc->parent; - GNUNET_CONTAINER_DLL_insert (rc->parent->children_head, - rc->parent->children_tail, - chld); + if (NULL != rc->parent) + GNUNET_CONTAINER_DLL_insert (rc->parent->children_head, + rc->parent->children_tail, + chld); return GNUNET_OK; } @@ -282,6 +283,10 @@ preprocess_file (const char *filename, free_tree (item); return GNUNET_SYSERR; } + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, + "..", 3)) + return GNUNET_SYSERR; } *dst = item; return GNUNET_OK; @@ -379,7 +384,8 @@ int main(int argc, } filename_expanded = argv[1]; ex = argv[2]; - if (0 != strcmp (ex, "-")) + if ( (ex == NULL) || + (0 != strcmp (ex, "-")) ) { plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); if (NULL != ex) diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c index 05bd66e31..9cc55a290 100644 --- a/src/fs/gnunet-publish.c +++ b/src/fs/gnunet-publish.c @@ -425,50 +425,67 @@ directory_trim_complete () } +/** + * Function called by the directory scanner as we build the tree + * that we will need to publish later. + * + * @param cls closure + * @param filename which file we are making progress on + * @param is_directory GNUNET_YES if this is a directory, + * GNUNET_NO if this is a file + * GNUNET_SYSERR if it is neither (or unknown) + * @param reason kind of progress we are making + */ static void -directory_scan_cb (void *cls, struct GNUNET_FS_DirScanner *ds, +directory_scan_cb (void *cls, const char *filename, int is_directory, enum GNUNET_FS_DirScannerProgressUpdateReason reason) { switch (reason) { - case GNUNET_FS_DIRSCANNER_FILE_START: + case GNUNET_FS_DIRSCANNER_FILE_START: + if (verbose > 1) + { if (is_directory) FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename); else FPRINTF (stdout, _("Scanning file `%s'.\n"), filename); - break; - case GNUNET_FS_DIRSCANNER_ALL_COUNTED: + } + break; + case GNUNET_FS_DIRSCANNER_FILE_IGNORED: + FPRINTF (stderr, + _("There was trouble processing file `%s', skipping it.\n"), + filename); + break; + case GNUNET_FS_DIRSCANNER_ALL_COUNTED: + if (verbose) FPRINTF (stdout, "%s", _("Preprocessing complete.\n")); - break; - case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED: + break; + case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED: + if (verbose > 2) FPRINTF (stdout, _("Extracting meta data from file `%s' complete.\n"), filename); - break; - case GNUNET_FS_DIRSCANNER_DOES_NOT_EXIST: - FPRINTF (stdout, - _("There was trouble processing file `%s', skipping it.\n"), - filename); - break; - case GNUNET_FS_DIRSCANNER_FINISHED: - FPRINTF (stdout, "%s", _("Scanner has finished.\n")); - directory_scan_result = GNUNET_FS_directory_scan_get_result (ds); - ds = NULL; - GNUNET_FS_share_tree_trim (directory_scan_result); - directory_trim_complete (); - break; - case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: - FPRINTF (stdout, "%s", _("Internal error scanning directory.\n")); - GNUNET_FS_directory_scan_abort (ds); - ds = NULL; - if (namespace != NULL) - GNUNET_FS_namespace_delete (namespace, GNUNET_NO); - GNUNET_FS_stop (ctx); - ret = 1; - break; - default: - GNUNET_assert (0); - break; + break; + case GNUNET_FS_DIRSCANNER_FINISHED: + if (verbose > 1) + FPRINTF (stdout, "%s", _("Meta data extraction has finished.\n")); + directory_scan_result = GNUNET_FS_directory_scan_get_result (ds); + ds = NULL; + GNUNET_FS_share_tree_trim (directory_scan_result); + directory_trim_complete (); + break; + case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: + FPRINTF (stdout, "%s", _("Internal error scanning directory.\n")); + GNUNET_FS_directory_scan_abort (ds); + ds = NULL; + if (namespace != NULL) + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + GNUNET_FS_stop (ctx); + ret = 1; + break; + default: + GNUNET_assert (0); + break; } fflush (stdout); } @@ -587,6 +604,12 @@ run (void *cls, char *const *args, const char *cfgfile, disable_extractor, ex, &directory_scan_cb, NULL); + if (NULL == ds) + { + FPRINTF (stderr, + "%s", _("Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n")); + return; + } kill_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task, NULL); -- 2.25.1