From: Christian Grothoff Date: Sun, 29 Jan 2012 20:21:09 +0000 (+0000) Subject: -finishing new helper X-Git-Tag: initial-import-from-subversion-38251~15021 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=6f912718b1e65181e9e3a62c6c7b8cf32f7d1165;p=oweals%2Fgnunet.git -finishing new helper --- diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c index e4ede571d..6d97f1553 100644 --- a/src/fs/fs_dirmetascan.c +++ b/src/fs/fs_dirmetascan.c @@ -149,6 +149,19 @@ process_helper_msgs (void *cls, is_directory, reason); GNUNET_free (filename); + + + /* having full filenames is too dangerous; always make sure we clean them up */ + item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); + + GNUNET_CONTAINER_meta_data_delete (item->meta, + EXTRACTOR_METATYPE_FILENAME, + NULL, 0); + GNUNET_CONTAINER_meta_data_insert (item->meta, "", + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + item->short_filename, + strlen (item->short_filename) + 1); #endif } diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c index 5edc2ec36..21ec19fe9 100644 --- a/src/fs/gnunet-helper-fs-publish.c +++ b/src/fs/gnunet-helper-fs-publish.c @@ -29,24 +29,94 @@ #include "platform.h" #include "gnunet_fs_service.h" + +/** + * A node of a directory tree. + */ +struct ScanTreeNode +{ + + /** + * This is a doubly-linked list + */ + struct ScanTreeNode *next; + + /** + * This is a doubly-linked list + */ + struct ScanTreeNode *prev; + + /** + * Parent of this node, NULL for top-level entries. + */ + struct ScanTreeNode *parent; + + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct ScanTreeNode *children_head; + + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct ScanTreeNode *children_tail; + + /** + * Name of the file/directory + */ + char *filename; + + /** + * Size of the file (if it is a file), in bytes + */ + uint64_t file_size; + + /** + * GNUNET_YES if this is a directory + */ + int is_directory; + +}; + + /** * List of libextractor plugins to use for extracting. */ static struct EXTRACTOR_PluginList *plugins; -#if 0 +/** + * Free memory of the 'tree' structure + * + * @param tree tree to free + */ +static void +free_tree (struct ScanTreeNode *tree) +{ + struct ScanTreeNode *pos; + + while (NULL != (pos = tree->children_head)) + free_tree (pos); + if (NULL != tree->parent) + GNUNET_CONTAINER_DLL_remove (tree->parent->children_head, + tree->parent->children_tail, + tree); + GNUNET_free (tree->filename); + GNUNET_free (tree); +} + + /** * Write 'size' bytes from 'buf' into 'out'. * - * @param in pipe to write to * @param buf buffer with data to write * @param size number of bytes to write * @return GNUNET_OK on success, GNUNET_SYSERR on error */ static int -write_all (const struct GNUNET_DISK_FileHandle *out, - const void *buf, +write_all (const void *buf, size_t size) { const char *cbuf = buf; @@ -56,54 +126,43 @@ write_all (const struct GNUNET_DISK_FileHandle *out, total = 0; do { - wr = GNUNET_DISK_file_write (out, - &cbuf[total], - size - total); + wr = write (1, + &cbuf[total], + size - total); if (wr > 0) total += wr; } while ( (wr > 0) && (total < size) ); if (wr <= 0) GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Failed to write to inter thread communication pipe: %s\n", + "Failed to write to stdout: %s\n", strerror (errno)); return (total == size) ? GNUNET_OK : GNUNET_SYSERR; } /** - * Write progress message. + * Write message to the master process. * - * @param ds - * @param filename name of the file to transmit, never NULL - * @param is_directory GNUNET_YES for directory, GNUNET_NO for file, GNUNET_SYSERR for neither - * @param reason reason for the progress call + * @param message_type message type to use + * @param data data to append, NULL for none + * @param data_length number of bytes in data * @return GNUNET_SYSERR to stop scanning (the pipe was broken somehow) */ static int -write_progress (struct GNUNET_FS_DirScanner *ds, - const char *filename, - int is_directory, - enum GNUNET_FS_DirScannerProgressUpdateReason reason) +write_message (uint16_t message_type, + const char *data, + size_t data_length) { - size_t slen; + struct GNUNET_MessageHeader hdr; - slen = strlen (filename) + 1; + hdr.type = htons (message_type); + hdr.size = htons (sizeof (struct GNUNET_MessageHeader) + data_length); if ( (GNUNET_OK != - write_all (ds->progress_write, - &reason, - sizeof (reason))) || - (GNUNET_OK != - write_all (ds->progress_write, - &slen, - sizeof (slen))) || - (GNUNET_OK != - write_all (ds->progress_write, - filename, - slen)) || + write_all (&hdr, + sizeof (hdr))) || (GNUNET_OK != - write_all (ds->progress_write, - &is_directory, - sizeof (is_directory))) ) + write_all (data, + data_length)) ) return GNUNET_SYSERR; return GNUNET_OK; } @@ -114,15 +173,13 @@ write_progress (struct GNUNET_FS_DirScanner *ds, * directory to the tree. Called by the directory scanner to initiate * the scan. Does NOT yet add any metadata. * - * @param ds directory scanner context to use * @param filename file or directory to scan * @param dst where to store the resulting share tree item * @return GNUNET_OK on success, GNUNET_SYSERR on error */ static int -preprocess_file (struct GNUNET_FS_DirScanner *ds, - const char *filename, - struct GNUNET_FS_ShareTreeItem **dst); +preprocess_file (const char *filename, + struct ScanTreeNode **dst); /** @@ -130,15 +187,10 @@ preprocess_file (struct GNUNET_FS_DirScanner *ds, */ struct RecursionContext { - /** - * Global scanner context. - */ - struct GNUNET_FS_DirScanner *ds; - /** * Parent to add the files to. */ - struct GNUNET_FS_ShareTreeItem *parent; + struct ScanTreeNode *parent; /** * Flag to set to GNUNET_YES on serious errors. @@ -161,11 +213,10 @@ scan_callback (void *cls, const char *filename) { struct RecursionContext *rc = cls; - struct GNUNET_FS_ShareTreeItem *chld; + struct ScanTreeNode *chld; if (GNUNET_OK != - preprocess_file (rc->ds, - filename, + preprocess_file (filename, &chld)) { rc->stop = GNUNET_YES; @@ -184,17 +235,15 @@ scan_callback (void *cls, * directory to the tree. Called by the directory scanner to initiate * the scan. Does NOT yet add any metadata. * - * @param ds directory scanner context to use * @param filename file or directory to scan * @param dst where to store the resulting share tree item * @return GNUNET_OK on success, GNUNET_SYSERR on error */ static int -preprocess_file (struct GNUNET_FS_DirScanner *ds, - const char *filename, - struct GNUNET_FS_ShareTreeItem **dst) +preprocess_file (const char *filename, + struct ScanTreeNode **dst) { - struct GNUNET_FS_ShareTreeItem *item; + struct ScanTreeNode *item; struct stat sbuf; if (0 != STAT (filename, &sbuf)) @@ -202,23 +251,21 @@ preprocess_file (struct GNUNET_FS_DirScanner *ds, /* If the file doesn't exist (or is not stat-able for any other reason) skip it (but report it), but do continue. */ if (GNUNET_OK != - write_progress (ds, filename, GNUNET_SYSERR, - GNUNET_FS_DIRSCANNER_DOES_NOT_EXIST)) + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, + filename, strlen (filename) + 1)) return GNUNET_SYSERR; return GNUNET_OK; } /* Report the progress */ if (GNUNET_OK != - write_progress (ds, - filename, - S_ISDIR (sbuf.st_mode) ? GNUNET_YES : GNUNET_NO, - GNUNET_FS_DIRSCANNER_FILE_START)) + write_message (S_ISDIR (sbuf.st_mode) + ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY + : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, + filename, strlen (filename) + 1)) return GNUNET_SYSERR; - item = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem)); - item->meta = GNUNET_CONTAINER_meta_data_create (); + item = GNUNET_malloc (sizeof (struct ScanTreeNode)); item->filename = GNUNET_strdup (filename); - item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name (filename)); item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO; item->file_size = (uint64_t) sbuf.st_size; if (item->is_directory) @@ -226,29 +273,16 @@ preprocess_file (struct GNUNET_FS_DirScanner *ds, struct RecursionContext rc; rc.parent = item; - rc.ds = ds; rc.stop = GNUNET_NO; GNUNET_DISK_directory_scan (filename, &scan_callback, &rc); - if ( (rc.stop == GNUNET_YES) || - (GNUNET_OK != - test_thread_stop (ds)) ) + if (rc.stop == GNUNET_YES) { - GNUNET_FS_share_tree_free (item); + free_tree (item); return GNUNET_SYSERR; } } - /* Report the progress */ - if (GNUNET_OK != - write_progress (ds, - filename, - S_ISDIR (sbuf.st_mode) ? GNUNET_YES : GNUNET_NO, - GNUNET_FS_DIRSCANNER_SUBTREE_COUNTED)) - { - GNUNET_FS_share_tree_free (item); - return GNUNET_SYSERR; - } *dst = item; return GNUNET_OK; } @@ -262,67 +296,85 @@ preprocess_file (struct GNUNET_FS_DirScanner *ds, * @return GNUNET_OK on success, GNUNET_SYSERR on fatal errors */ static int -extract_files (struct GNUNET_FS_DirScanner *ds, - struct GNUNET_FS_ShareTreeItem *item) +extract_files (struct ScanTreeNode *item) { + struct GNUNET_CONTAINER_MetaData *meta; + ssize_t size; + size_t slen; + if (item->is_directory) { /* for directories, we simply only descent, no extraction, no progress reporting */ - struct GNUNET_FS_ShareTreeItem *pos; + struct ScanTreeNode *pos; for (pos = item->children_head; NULL != pos; pos = pos->next) if (GNUNET_OK != - extract_files (ds, pos)) + extract_files (pos)) return GNUNET_SYSERR; return GNUNET_OK; } /* this is the expensive operation, *afterwards* we'll check for aborts */ - fprintf (stderr, "\tCalling extract on `%s'\n", item->filename); - GNUNET_FS_meta_data_extract_from_file (item->meta, + meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_FS_meta_data_extract_from_file (meta, item->filename, - ds->plugins); - fprintf (stderr, "\tExtract `%s' done\n", item->filename); - - /* having full filenames is too dangerous; always make sure we clean them up */ - GNUNET_CONTAINER_meta_data_delete (item->meta, - EXTRACTOR_METATYPE_FILENAME, - NULL, 0); - GNUNET_CONTAINER_meta_data_insert (item->meta, "", - EXTRACTOR_METATYPE_FILENAME, - EXTRACTOR_METAFORMAT_UTF8, "text/plain", - item->short_filename, - strlen (item->short_filename) + 1); - /* check for abort */ - if (GNUNET_OK != - test_thread_stop (ds)) - return GNUNET_SYSERR; - - /* Report the progress */ - if (GNUNET_OK != - write_progress (ds, - item->filename, - GNUNET_NO, - GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED)) - return GNUNET_SYSERR; + plugins); + slen = strlen (item->filename) + 1; + size = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); + if ( (-1 == size) || + (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE - slen) ) + { + /* no meta data */ + GNUNET_CONTAINER_meta_data_destroy (meta); + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, + item->filename, slen)) + return GNUNET_SYSERR; + return GNUNET_OK; + } + { + char buf[size + slen]; + char *dst = buf; + + memcpy (buf, item->filename, slen); + size = GNUNET_CONTAINER_meta_data_serialize (meta, + &dst, size, + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + GNUNET_CONTAINER_meta_data_destroy (meta); + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, + buf, + slen + size)) + return GNUNET_SYSERR; + } return GNUNET_OK; } -#endif - +/** + * Main function of the helper process to extract meta data. + * + * @param argc should be 3 + * @param argv [0] our binary name + * [1] name of the file or directory to process + * [2] "-" to disable extraction, NULL for defaults, + * otherwise custom plugins to load from LE + * @return 0 on success + */ int main(int argc, char **argv) { const char *filename_expanded; const char *ex; + struct ScanTreeNode *root; - if (argc < 3) + /* parse command line */ + if ( (argc != 3) && (argc != 2) ) { FPRINTF (stderr, "%s", - "gnunet-helper-fs-publish needs at least two arguments\n"); + "gnunet-helper-fs-publish needs exactly one or two arguments\n"); return 1; } filename_expanded = argv[1]; @@ -335,29 +387,33 @@ int main(int argc, EXTRACTOR_OPTION_DEFAULT_POLICY); } -#if 0 + /* scan tree to find out how much work there is to be done */ if (GNUNET_OK != preprocess_file (filename_expanded, - &toplevel)) + &root)) { - (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); - GNUNET_DISK_pipe_close_end (ds->progress_pipe, GNUNET_DISK_PIPE_END_WRITE); + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0); return 2; } + /* signal that we're done counting files, so that a percentage of + progress can now be calculated */ if (GNUNET_OK != - write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_ALL_COUNTED)) - { - return 3; - } + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, NULL, 0)) + return 3; if (GNUNET_OK != - extract_files (ds, ds->toplevel)) + extract_files (root)) { - (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0); + free_tree (root); return 4; } - (void) write_progress (ds, "", GNUNET_SYSERR, GNUNET_FS_DIRSCANNER_FINISHED); -#endif + free_tree (root); + /* enable "clean" shutdown by telling parent that we are done */ + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, NULL, 0); if (NULL != plugins) EXTRACTOR_plugin_remove_all (plugins); return 0; } + +/* end of gnunet-helper-fs-publish.c */ + diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c index 15054a722..05bd66e31 100644 --- a/src/fs/gnunet-publish.c +++ b/src/fs/gnunet-publish.c @@ -439,10 +439,6 @@ directory_scan_cb (void *cls, struct GNUNET_FS_DirScanner *ds, else FPRINTF (stdout, _("Scanning file `%s'.\n"), filename); break; - case GNUNET_FS_DIRSCANNER_SUBTREE_COUNTED: - if (is_directory) - FPRINTF (stdout, _("Done scanning directory `%s'.\n"), filename); - break; case GNUNET_FS_DIRSCANNER_ALL_COUNTED: FPRINTF (stdout, "%s", _("Preprocessing complete.\n")); break;