2 This file is part of GNUnet
3 Copyright (C) 2005-2012 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
17 * @file fs/fs_dirmetascan.c
18 * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem'
19 * from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.
21 * @author Christian Grothoff
24 #include "gnunet_fs_service.h"
25 #include "gnunet_scheduler_lib.h"
30 * An opaque structure a pointer to which is returned to the
31 * caller to be used to control the scanner.
33 struct GNUNET_FS_DirScanner
39 struct GNUNET_HELPER_Handle *helper;
42 * Expanded filename (as given by the scan initiator).
43 * The scanner thread stores a copy here, and frees it when it finishes.
45 char *filename_expanded;
48 * Second argument to helper process.
53 * The function that will be called every time there's a progress
56 GNUNET_FS_DirScannerProgressCallback progress_callback;
59 * A closure for progress_callback.
61 void *progress_callback_cls;
64 * After the scan is finished, it will contain a pointer to the
65 * top-level directory entry in the directory tree built by the
68 struct GNUNET_FS_ShareTreeItem *toplevel;
71 * Current position during processing.
73 struct GNUNET_FS_ShareTreeItem *pos;
76 * Task scheduled when we are done.
78 struct GNUNET_SCHEDULER_Task * stop_task;
81 * Arguments for helper.
89 * Abort the scan. Must not be called from within the progress_callback
92 * @param ds directory scanner structure
95 GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
97 /* terminate helper */
98 if (NULL != ds->helper)
99 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
102 if (NULL != ds->toplevel)
103 GNUNET_FS_share_tree_free (ds->toplevel);
104 if (NULL != ds->stop_task)
105 GNUNET_SCHEDULER_cancel (ds->stop_task);
106 GNUNET_free_non_null (ds->ex_arg);
107 GNUNET_free (ds->filename_expanded);
113 * Obtain the result of the scan after the scan has signalled
114 * completion. Must not be called prior to completion. The 'ds' is
115 * freed as part of this call.
117 * @param ds directory scanner structure
118 * @return the results of the scan (a directory tree)
120 struct GNUNET_FS_ShareTreeItem *
121 GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
123 struct GNUNET_FS_ShareTreeItem *result;
125 /* check that we're actually done */
126 GNUNET_assert (NULL == ds->helper);
127 /* preserve result */
128 result = ds->toplevel;
130 GNUNET_FS_directory_scan_abort (ds);
136 * Move in the directory from the given position to the next file
139 * @param pos current position
140 * @return next file, NULL for none
142 static struct GNUNET_FS_ShareTreeItem *
143 advance (struct GNUNET_FS_ShareTreeItem *pos)
147 GNUNET_assert (NULL != pos);
148 moved = 0; /* must not terminate, even on file, otherwise "normal" */
149 while ( (pos->is_directory == GNUNET_YES) ||
152 if ( (moved != -1) &&
153 (NULL != pos->children_head) )
155 pos = pos->children_head;
156 moved = 1; /* can terminate if file */
159 if (NULL != pos->next)
162 moved = 1; /* can terminate if file */
165 if (NULL != pos->parent)
168 moved = -1; /* force move to 'next' or 'parent' */
171 /* no more options, end of traversal */
179 * Add another child node to the tree.
181 * @param parent parent of the child, NULL for top level
182 * @param filename name of the file or directory
183 * @param is_directory GNUNET_YES for directories
184 * @return new entry that was just created
186 static struct GNUNET_FS_ShareTreeItem *
187 expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
188 const char *filename,
191 struct GNUNET_FS_ShareTreeItem *chld;
194 chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
195 chld->parent = parent;
196 chld->filename = GNUNET_strdup (filename);
197 GNUNET_asprintf (&chld->short_filename,
199 GNUNET_STRINGS_get_short_name (filename),
200 is_directory == GNUNET_YES ? "/" : "");
201 /* make sure we do not end with '//' */
202 slen = strlen (chld->short_filename);
204 (chld->short_filename[slen-1] == '/') &&
205 (chld->short_filename[slen-2] == '/') )
206 chld->short_filename[slen-1] = '\0';
207 chld->is_directory = is_directory;
209 GNUNET_CONTAINER_DLL_insert (parent->children_head,
210 parent->children_tail,
217 * Task run last to shut everything down.
219 * @param cls the 'struct GNUNET_FS_DirScanner'
222 finish_scan (void *cls)
224 struct GNUNET_FS_DirScanner *ds = cls;
226 ds->stop_task = NULL;
227 if (NULL != ds->helper)
229 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
232 ds->progress_callback (ds->progress_callback_cls,
234 GNUNET_FS_DIRSCANNER_FINISHED);
239 * Called every time there is data to read from the scanner.
240 * Calls the scanner progress handler.
242 * @param cls the closure (directory scanner object)
243 * @param msg message from the helper process
244 * @return #GNUNET_OK on success,
245 * #GNUNET_NO to stop further processing (no error)
246 * #GNUNET_SYSERR to stop further processing with error
249 process_helper_msgs (void *cls,
250 const struct GNUNET_MessageHeader *msg)
252 struct GNUNET_FS_DirScanner *ds = cls;
253 const char *filename;
258 "DMS parses %u-byte message of type %u\n",
259 (unsigned int) ntohs (msg->size),
260 (unsigned int) ntohs (msg->type));
262 left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
263 filename = (const char*) &msg[1];
264 switch (ntohs (msg->type))
266 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
267 if (filename[left-1] != '\0')
272 ds->progress_callback (ds->progress_callback_cls,
274 GNUNET_FS_DIRSCANNER_FILE_START);
275 if (NULL == ds->toplevel)
277 ds->toplevel = expand_tree (ds->pos,
283 GNUNET_assert (NULL != ds->pos);
284 (void) expand_tree (ds->pos,
289 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
290 if (filename[left-1] != '\0')
295 if (0 == strcmp ("..", filename))
302 ds->pos = ds->pos->parent;
305 ds->progress_callback (ds->progress_callback_cls,
306 filename, GNUNET_YES,
307 GNUNET_FS_DIRSCANNER_FILE_START);
308 ds->pos = expand_tree (ds->pos,
311 if (NULL == ds->toplevel)
312 ds->toplevel = ds->pos;
314 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
316 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
317 if ('\0' != filename[left-1])
319 ds->progress_callback (ds->progress_callback_cls,
320 filename, GNUNET_SYSERR,
321 GNUNET_FS_DIRSCANNER_FILE_IGNORED);
323 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
329 if (NULL == ds->toplevel)
334 ds->progress_callback (ds->progress_callback_cls,
336 GNUNET_FS_DIRSCANNER_ALL_COUNTED);
337 ds->pos = ds->toplevel;
338 if (GNUNET_YES == ds->pos->is_directory)
339 ds->pos = advance (ds->pos);
341 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA:
351 end = memchr (filename, 0, left);
358 nlen = end - filename;
360 if (0 != strcmp (filename,
366 ds->progress_callback (ds->progress_callback_cls,
369 GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
372 ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end,
374 if (NULL == ds->pos->meta)
379 /* having full filenames is too dangerous; always make sure we clean them up */
380 GNUNET_CONTAINER_meta_data_delete (ds->pos->meta,
381 EXTRACTOR_METATYPE_FILENAME,
383 /* instead, put in our 'safer' original filename */
384 GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
385 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
386 EXTRACTOR_METAFORMAT_UTF8, "text/plain",
387 ds->pos->short_filename,
388 strlen (ds->pos->short_filename) + 1);
390 ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (ds->pos->meta);
391 ds->pos = advance (ds->pos);
394 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
405 if (NULL == ds->toplevel)
410 ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan,
417 ds->progress_callback (ds->progress_callback_cls,
419 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
425 * Function called if our helper process died.
427 * @param cls the 'struct GNUNET_FS_DirScanner' callback.
430 helper_died_cb (void *cls)
432 struct GNUNET_FS_DirScanner *ds = cls;
435 if (NULL != ds->stop_task)
436 return; /* normal death, was finished */
437 ds->progress_callback (ds->progress_callback_cls,
439 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
444 * Start a directory scanner thread.
446 * @param filename name of the directory to scan
447 * @param disable_extractor #GNUNET_YES to not run libextractor on files (only
449 * @param ex if not NULL, must be a list of extra plugins for extractor
450 * @param cb the callback to call when there are scanning progress messages
451 * @param cb_cls closure for 'cb'
452 * @return directory scanner object to be used for controlling the scanner
454 struct GNUNET_FS_DirScanner *
455 GNUNET_FS_directory_scan_start (const char *filename,
456 int disable_extractor, const char *ex,
457 GNUNET_FS_DirScannerProgressCallback cb,
461 char *filename_expanded;
462 struct GNUNET_FS_DirScanner *ds;
464 if (0 != STAT (filename, &sbuf))
466 filename_expanded = GNUNET_STRINGS_filename_expand (filename);
467 if (NULL == filename_expanded)
469 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
470 "Starting to scan directory `%s'\n",
472 ds = GNUNET_new (struct GNUNET_FS_DirScanner);
473 ds->progress_callback = cb;
474 ds->progress_callback_cls = cb_cls;
475 ds->filename_expanded = filename_expanded;
476 if (disable_extractor)
477 ds->ex_arg = GNUNET_strdup ("-");
479 ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
480 ds->args[0] = "gnunet-helper-fs-publish";
481 ds->args[1] = ds->filename_expanded;
482 ds->args[2] = ds->ex_arg;
484 ds->helper = GNUNET_HELPER_start (GNUNET_NO,
485 "gnunet-helper-fs-publish",
487 &process_helper_msgs,
488 &helper_died_cb, ds);
489 if (NULL == ds->helper)
491 GNUNET_free (filename_expanded);
499 /* end of fs_dirmetascan.c */