2 This file is part of GNUnet.
3 (C) 2012 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file src/fs/gnunet-helper-fs-publish.c
23 * @brief Tool to help extract meta data asynchronously
24 * @author Christian Grothoff
26 * This program will scan a directory for files with meta data
27 * and report the results to stdout.
30 #include "gnunet_fs_service.h"
34 * A node of a directory tree.
40 * This is a doubly-linked list
42 struct ScanTreeNode *next;
45 * This is a doubly-linked list
47 struct ScanTreeNode *prev;
50 * Parent of this node, NULL for top-level entries.
52 struct ScanTreeNode *parent;
55 * This is a doubly-linked tree
56 * NULL for files and empty directories
58 struct ScanTreeNode *children_head;
61 * This is a doubly-linked tree
62 * NULL for files and empty directories
64 struct ScanTreeNode *children_tail;
67 * Name of the file/directory
72 * Size of the file (if it is a file), in bytes.
73 * At the moment it is set to 0 for directories.
78 * #GNUNET_YES if this is a directory
86 * List of libextractor plugins to use for extracting.
88 static struct EXTRACTOR_PluginList *plugins;
91 * File descriptor we use for IPC with the parent.
93 static int output_stream;
97 * Add meta data that libextractor finds to our meta data
100 * @param cls closure, our meta data container
101 * @param plugin_name name of the plugin that produced this value;
102 * special values can be used (i.e. '<zlib>' for zlib being
103 * used in the main libextractor library and yielding
105 * @param type libextractor-type describing the meta data
106 * @param format basic format information about data
107 * @param data_mime_type mime-type of data (not of the original file);
108 * can be NULL (if mime-type is not known)
109 * @param data actual meta-data found
110 * @param data_len number of bytes in @a data
111 * @return always 0 to continue extracting
114 add_to_md (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type,
115 enum EXTRACTOR_MetaFormat format, const char *data_mime_type,
116 const char *data, size_t data_len)
118 struct GNUNET_CONTAINER_MetaData *md = cls;
120 if ( ((EXTRACTOR_METAFORMAT_UTF8 == format) ||
121 (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
122 ('\0' != data[data_len - 1]) )
124 char zdata[data_len + 1];
125 memcpy (zdata, data, data_len);
126 zdata[data_len] = '\0';
127 (void) GNUNET_CONTAINER_meta_data_insert (md, plugin_name, type, format,
128 data_mime_type, zdata, data_len + 1);
132 (void) GNUNET_CONTAINER_meta_data_insert (md, plugin_name, type, format,
133 data_mime_type, data, data_len);
140 * Free memory of the @a tree structure
142 * @param tree tree to free
145 free_tree (struct ScanTreeNode *tree)
147 struct ScanTreeNode *pos;
149 while (NULL != (pos = tree->children_head))
151 if (NULL != tree->parent)
152 GNUNET_CONTAINER_DLL_remove (tree->parent->children_head,
153 tree->parent->children_tail,
155 GNUNET_free (tree->filename);
161 * Write @a size bytes from @a buf into the #output_stream.
163 * @param buf buffer with data to write
164 * @param size number of bytes to write
165 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
168 write_all (const void *buf,
171 const char *cbuf = buf;
178 wr = write (output_stream,
183 } while ( (wr > 0) && (total < size) );
185 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
186 "Failed to write to stdout: %s\n",
188 return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
193 * Write message to the master process.
195 * @param message_type message type to use
196 * @param data data to append, NULL for none
197 * @param data_length number of bytes in @a data
198 * @return #GNUNET_SYSERR to stop scanning (the pipe was broken somehow)
201 write_message (uint16_t message_type,
205 struct GNUNET_MessageHeader hdr;
209 "Helper sends %u-byte message of type %u\n",
210 (unsigned int) (sizeof (struct GNUNET_MessageHeader) + data_length),
211 (unsigned int) message_type);
213 hdr.type = htons (message_type);
214 hdr.size = htons (sizeof (struct GNUNET_MessageHeader) + data_length);
221 return GNUNET_SYSERR;
227 * Function called to (recursively) add all of the files in the
228 * directory to the tree. Called by the directory scanner to initiate
229 * the scan. Does NOT yet add any metadata.
231 * @param filename file or directory to scan
232 * @param dst where to store the resulting share tree item;
233 * NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned)
234 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
237 preprocess_file (const char *filename,
238 struct ScanTreeNode **dst);
242 * Closure for the 'scan_callback'
244 struct RecursionContext
247 * Parent to add the files to.
249 struct ScanTreeNode *parent;
252 * Flag to set to GNUNET_YES on serious errors.
259 * Function called by the directory iterator to (recursively) add all
260 * of the files in the directory to the tree. Called by the directory
261 * scanner to initiate the scan. Does NOT yet add any metadata.
263 * @param cls the `struct RecursionContext`
264 * @param filename file or directory to scan
265 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
268 scan_callback (void *cls,
269 const char *filename)
271 struct RecursionContext *rc = cls;
272 struct ScanTreeNode *chld;
275 preprocess_file (filename,
278 rc->stop = GNUNET_YES;
279 return GNUNET_SYSERR;
283 chld->parent = rc->parent;
284 GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
285 rc->parent->children_tail,
292 * Function called to (recursively) add all of the files in the
293 * directory to the tree. Called by the directory scanner to initiate
294 * the scan. Does NOT yet add any metadata.
296 * @param filename file or directory to scan
297 * @param dst where to store the resulting share tree item;
298 * NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned)
299 * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
302 preprocess_file (const char *filename,
303 struct ScanTreeNode **dst)
305 struct ScanTreeNode *item;
309 if ((0 != STAT (filename, &sbuf)) ||
310 ((!S_ISDIR (sbuf.st_mode)) && (GNUNET_OK != GNUNET_DISK_file_size (
311 filename, &fsize, GNUNET_NO, GNUNET_YES))))
313 /* If the file doesn't exist (or is not stat-able for any other reason)
314 skip it (but report it), but do continue. */
316 write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE,
317 filename, strlen (filename) + 1))
318 return GNUNET_SYSERR;
319 /* recoverable error, store 'NULL' in *dst */
324 /* Report the progress */
326 write_message (S_ISDIR (sbuf.st_mode)
327 ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
328 : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE,
329 filename, strlen (filename) + 1))
330 return GNUNET_SYSERR;
331 item = GNUNET_malloc (sizeof (struct ScanTreeNode));
332 item->filename = GNUNET_strdup (filename);
333 item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
334 item->file_size = fsize;
335 if (GNUNET_YES == item->is_directory)
337 struct RecursionContext rc;
341 GNUNET_DISK_directory_scan (filename,
344 if ( (GNUNET_YES == rc.stop) ||
346 write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
350 return GNUNET_SYSERR;
359 * Extract metadata from files.
361 * @param item entry we are processing
362 * @return #GNUNET_OK on success, #GNUNET_SYSERR on fatal errors
365 extract_files (struct ScanTreeNode *item)
367 struct GNUNET_CONTAINER_MetaData *meta;
371 if (GNUNET_YES == item->is_directory)
373 /* for directories, we simply only descent, no extraction, no
374 progress reporting */
375 struct ScanTreeNode *pos;
377 for (pos = item->children_head; NULL != pos; pos = pos->next)
380 return GNUNET_SYSERR;
384 /* this is the expensive operation, *afterwards* we'll check for aborts */
385 meta = GNUNET_CONTAINER_meta_data_create ();
386 EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
387 slen = strlen (item->filename) + 1;
388 size = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
392 GNUNET_CONTAINER_meta_data_destroy (meta);
394 write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
395 item->filename, slen))
396 return GNUNET_SYSERR;
399 else if (size > (UINT16_MAX - sizeof (struct GNUNET_MessageHeader) - slen))
401 /* We can't transfer more than 64k bytes in one message. */
402 size = UINT16_MAX - sizeof (struct GNUNET_MessageHeader) - slen;
405 char buf[size + slen];
406 char *dst = &buf[slen];
408 memcpy (buf, item->filename, slen);
409 size = GNUNET_CONTAINER_meta_data_serialize (meta,
411 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
417 GNUNET_CONTAINER_meta_data_destroy (meta);
419 write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
422 return GNUNET_SYSERR;
430 * Install a signal handler to ignore SIGPIPE.
435 struct sigaction oldsig;
436 struct sigaction sig;
438 memset (&sig, 0, sizeof (struct sigaction));
439 sig.sa_handler = SIG_IGN;
440 sigemptyset (&sig.sa_mask);
442 sig.sa_flags = SA_INTERRUPT; /* SunOS */
444 sig.sa_flags = SA_RESTART;
446 if (0 != sigaction (SIGPIPE, &sig, &oldsig))
448 "Failed to install SIGPIPE handler: %s\n", strerror (errno));
453 * Turn the given file descriptor in to '/dev/null'.
455 * @param fd fd to bind to /dev/null
456 * @param flags flags to use (O_RDONLY or O_WRONLY)
459 make_dev_zero (int fd,
464 GNUNET_assert (0 == close (fd));
465 z = open ("/dev/null", flags);
466 GNUNET_assert (-1 != z);
470 GNUNET_assert (0 == close (z));
477 * Main function of the helper process to extract meta data.
479 * @param argc should be 3
480 * @param argv [0] our binary name
481 * [1] name of the file or directory to process
482 * [2] "-" to disable extraction, NULL for defaults,
483 * otherwise custom plugins to load from LE
484 * @return 0 on success
490 const char *filename_expanded;
492 struct ScanTreeNode *root;
495 /* We're using stdout to communicate binary data back to the parent; use
498 _setmode (1, _O_BINARY);
499 /* Get utf-8-encoded arguments */
500 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
502 output_stream = 1; /* stdout */
505 /* move stdout to some other FD for IPC, bind
506 stdout/stderr to /dev/null */
507 output_stream = dup (1);
508 make_dev_zero (1, O_WRONLY);
509 make_dev_zero (2, O_WRONLY);
512 /* parse command line */
513 if ( (3 != argc) && (2 != argc) )
517 "gnunet-helper-fs-publish needs exactly one or two arguments\n");
519 GNUNET_free ((void*) argv);
523 filename_expanded = argv[1];
526 (0 != strcmp (ex, "-")) )
528 plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
530 plugins = EXTRACTOR_plugin_add_config (plugins, ex,
531 EXTRACTOR_OPTION_DEFAULT_POLICY);
534 /* scan tree to find out how much work there is to be done */
535 if (GNUNET_OK != preprocess_file (filename_expanded,
538 (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0);
539 EXTRACTOR_plugin_remove_all (plugins);
541 GNUNET_free ((void*) argv);
545 /* signal that we're done counting files, so that a percentage of
546 progress can now be calculated */
548 write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, NULL, 0))
550 EXTRACTOR_plugin_remove_all (plugins);
552 GNUNET_free ((void*) argv);
559 extract_files (root))
561 (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0);
563 EXTRACTOR_plugin_remove_all (plugins);
565 GNUNET_free ((void*) argv);
571 /* enable "clean" shutdown by telling parent that we are done */
572 (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, NULL, 0);
573 EXTRACTOR_plugin_remove_all (plugins);
575 GNUNET_free ((void*) argv);
580 /* end of gnunet-helper-fs-publish.c */