2 This file is part of GNUnet
3 Copyright (C) 2005-2012 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
22 * @file fs/fs_dirmetascan.c
23 * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem'
24 * from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.
26 * @author Christian Grothoff
29 #include "gnunet_fs_service.h"
30 #include "gnunet_scheduler_lib.h"
35 * An opaque structure a pointer to which is returned to the
36 * caller to be used to control the scanner.
38 struct GNUNET_FS_DirScanner
44 struct GNUNET_HELPER_Handle *helper;
47 * Expanded filename (as given by the scan initiator).
48 * The scanner thread stores a copy here, and frees it when it finishes.
50 char *filename_expanded;
53 * Second argument to helper process.
58 * The function that will be called every time there's a progress
61 GNUNET_FS_DirScannerProgressCallback progress_callback;
64 * A closure for progress_callback.
66 void *progress_callback_cls;
69 * After the scan is finished, it will contain a pointer to the
70 * top-level directory entry in the directory tree built by the
73 struct GNUNET_FS_ShareTreeItem *toplevel;
76 * Current position during processing.
78 struct GNUNET_FS_ShareTreeItem *pos;
81 * Task scheduled when we are done.
83 struct GNUNET_SCHEDULER_Task * stop_task;
86 * Arguments for helper.
94 * Abort the scan. Must not be called from within the progress_callback
97 * @param ds directory scanner structure
100 GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
102 /* terminate helper */
103 if (NULL != ds->helper)
104 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
107 if (NULL != ds->toplevel)
108 GNUNET_FS_share_tree_free (ds->toplevel);
109 if (NULL != ds->stop_task)
110 GNUNET_SCHEDULER_cancel (ds->stop_task);
111 GNUNET_free_non_null (ds->ex_arg);
112 GNUNET_free (ds->filename_expanded);
118 * Obtain the result of the scan after the scan has signalled
119 * completion. Must not be called prior to completion. The 'ds' is
120 * freed as part of this call.
122 * @param ds directory scanner structure
123 * @return the results of the scan (a directory tree)
125 struct GNUNET_FS_ShareTreeItem *
126 GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
128 struct GNUNET_FS_ShareTreeItem *result;
130 /* check that we're actually done */
131 GNUNET_assert (NULL == ds->helper);
132 /* preserve result */
133 result = ds->toplevel;
135 GNUNET_FS_directory_scan_abort (ds);
141 * Move in the directory from the given position to the next file
144 * @param pos current position
145 * @return next file, NULL for none
147 static struct GNUNET_FS_ShareTreeItem *
148 advance (struct GNUNET_FS_ShareTreeItem *pos)
152 GNUNET_assert (NULL != pos);
153 moved = 0; /* must not terminate, even on file, otherwise "normal" */
154 while ( (pos->is_directory == GNUNET_YES) ||
157 if ( (moved != -1) &&
158 (NULL != pos->children_head) )
160 pos = pos->children_head;
161 moved = 1; /* can terminate if file */
164 if (NULL != pos->next)
167 moved = 1; /* can terminate if file */
170 if (NULL != pos->parent)
173 moved = -1; /* force move to 'next' or 'parent' */
176 /* no more options, end of traversal */
184 * Add another child node to the tree.
186 * @param parent parent of the child, NULL for top level
187 * @param filename name of the file or directory
188 * @param is_directory GNUNET_YES for directories
189 * @return new entry that was just created
191 static struct GNUNET_FS_ShareTreeItem *
192 expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
193 const char *filename,
196 struct GNUNET_FS_ShareTreeItem *chld;
199 chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
200 chld->parent = parent;
201 chld->filename = GNUNET_strdup (filename);
202 GNUNET_asprintf (&chld->short_filename,
204 GNUNET_STRINGS_get_short_name (filename),
205 is_directory == GNUNET_YES ? "/" : "");
206 /* make sure we do not end with '//' */
207 slen = strlen (chld->short_filename);
209 (chld->short_filename[slen-1] == '/') &&
210 (chld->short_filename[slen-2] == '/') )
211 chld->short_filename[slen-1] = '\0';
212 chld->is_directory = is_directory;
214 GNUNET_CONTAINER_DLL_insert (parent->children_head,
215 parent->children_tail,
222 * Task run last to shut everything down.
224 * @param cls the 'struct GNUNET_FS_DirScanner'
227 finish_scan (void *cls)
229 struct GNUNET_FS_DirScanner *ds = cls;
231 ds->stop_task = NULL;
232 if (NULL != ds->helper)
234 GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
237 ds->progress_callback (ds->progress_callback_cls,
239 GNUNET_FS_DIRSCANNER_FINISHED);
244 * Called every time there is data to read from the scanner.
245 * Calls the scanner progress handler.
247 * @param cls the closure (directory scanner object)
248 * @param msg message from the helper process
249 * @return #GNUNET_OK on success,
250 * #GNUNET_NO to stop further processing (no error)
251 * #GNUNET_SYSERR to stop further processing with error
254 process_helper_msgs (void *cls,
255 const struct GNUNET_MessageHeader *msg)
257 struct GNUNET_FS_DirScanner *ds = cls;
258 const char *filename;
263 "DMS parses %u-byte message of type %u\n",
264 (unsigned int) ntohs (msg->size),
265 (unsigned int) ntohs (msg->type));
267 left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
268 filename = (const char*) &msg[1];
269 switch (ntohs (msg->type))
271 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
272 if (filename[left-1] != '\0')
277 ds->progress_callback (ds->progress_callback_cls,
279 GNUNET_FS_DIRSCANNER_FILE_START);
280 if (NULL == ds->toplevel)
282 ds->toplevel = expand_tree (ds->pos,
288 GNUNET_assert (NULL != ds->pos);
289 (void) expand_tree (ds->pos,
294 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
295 if (filename[left-1] != '\0')
300 if (0 == strcmp ("..", filename))
307 ds->pos = ds->pos->parent;
310 ds->progress_callback (ds->progress_callback_cls,
311 filename, GNUNET_YES,
312 GNUNET_FS_DIRSCANNER_FILE_START);
313 ds->pos = expand_tree (ds->pos,
316 if (NULL == ds->toplevel)
317 ds->toplevel = ds->pos;
319 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
321 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
322 if ('\0' != filename[left-1])
324 ds->progress_callback (ds->progress_callback_cls,
325 filename, GNUNET_SYSERR,
326 GNUNET_FS_DIRSCANNER_FILE_IGNORED);
328 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
334 if (NULL == ds->toplevel)
339 ds->progress_callback (ds->progress_callback_cls,
341 GNUNET_FS_DIRSCANNER_ALL_COUNTED);
342 ds->pos = ds->toplevel;
343 if (GNUNET_YES == ds->pos->is_directory)
344 ds->pos = advance (ds->pos);
346 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA:
356 end = memchr (filename, 0, left);
363 nlen = end - filename;
365 if (0 != strcmp (filename,
371 ds->progress_callback (ds->progress_callback_cls,
374 GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
377 ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end,
379 if (NULL == ds->pos->meta)
384 /* having full filenames is too dangerous; always make sure we clean them up */
385 GNUNET_CONTAINER_meta_data_delete (ds->pos->meta,
386 EXTRACTOR_METATYPE_FILENAME,
388 /* instead, put in our 'safer' original filename */
389 GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
390 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
391 EXTRACTOR_METAFORMAT_UTF8, "text/plain",
392 ds->pos->short_filename,
393 strlen (ds->pos->short_filename) + 1);
395 ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (ds->pos->meta);
396 ds->pos = advance (ds->pos);
399 case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
410 if (NULL == ds->toplevel)
415 ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan,
422 ds->progress_callback (ds->progress_callback_cls,
424 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
430 * Function called if our helper process died.
432 * @param cls the 'struct GNUNET_FS_DirScanner' callback.
435 helper_died_cb (void *cls)
437 struct GNUNET_FS_DirScanner *ds = cls;
440 if (NULL != ds->stop_task)
441 return; /* normal death, was finished */
442 ds->progress_callback (ds->progress_callback_cls,
444 GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
449 * Start a directory scanner thread.
451 * @param filename name of the directory to scan
452 * @param disable_extractor #GNUNET_YES to not run libextractor on files (only
454 * @param ex if not NULL, must be a list of extra plugins for extractor
455 * @param cb the callback to call when there are scanning progress messages
456 * @param cb_cls closure for 'cb'
457 * @return directory scanner object to be used for controlling the scanner
459 struct GNUNET_FS_DirScanner *
460 GNUNET_FS_directory_scan_start (const char *filename,
461 int disable_extractor, const char *ex,
462 GNUNET_FS_DirScannerProgressCallback cb,
466 char *filename_expanded;
467 struct GNUNET_FS_DirScanner *ds;
469 if (0 != STAT (filename, &sbuf))
471 filename_expanded = GNUNET_STRINGS_filename_expand (filename);
472 if (NULL == filename_expanded)
474 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
475 "Starting to scan directory `%s'\n",
477 ds = GNUNET_new (struct GNUNET_FS_DirScanner);
478 ds->progress_callback = cb;
479 ds->progress_callback_cls = cb_cls;
480 ds->filename_expanded = filename_expanded;
481 if (disable_extractor)
482 ds->ex_arg = GNUNET_strdup ("-");
484 ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
485 ds->args[0] = "gnunet-helper-fs-publish";
486 ds->args[1] = ds->filename_expanded;
487 ds->args[2] = ds->ex_arg;
489 ds->helper = GNUNET_HELPER_start (GNUNET_NO,
490 "gnunet-helper-fs-publish",
492 &process_helper_msgs,
493 &helper_died_cb, ds);
494 if (NULL == ds->helper)
496 GNUNET_free (filename_expanded);
504 /* end of fs_dirmetascan.c */