2 This file is part of GNUnet
3 (C) 2005-2012 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 #include "gnunet_fs_service.h"
23 #include "gnunet_scheduler_lib.h"
27 * Entry for each unique keyword to track how often
28 * it occured. Contains the keyword and the counter.
34 * Keyword that was found.
39 * How many files have this keyword?
44 * This is a doubly-linked list
46 struct KeywordCounter *prev;
49 * This is a doubly-linked list
51 struct KeywordCounter *next;
55 * Aggregate information we keep for meta data in each directory.
60 * The actual meta data.
65 * Number of bytes in 'data'.
70 * Name of the plugin that provided that piece of metadata
72 const char *plugin_name;
77 enum EXTRACTOR_MetaType type;
82 enum EXTRACTOR_MetaFormat format;
85 * MIME-type of the metadata itself
87 const char *data_mime_type;
90 * How many files have meta entries matching this value?
91 * (type and format do not have to match).
96 * This is a doubly-linked list
98 struct MetaCounter *prev;
101 * This is a doubly-linked list
103 struct MetaCounter *next;
106 struct AddDirContext;
109 * A structure used to hold a pointer to the tree item that is being
111 * Needed to avoid changing the context for every recursive call.
118 struct AddDirContext *adc;
123 struct GNUNET_FS_ShareTreeItem *parent;
127 * Execution context for 'add_dir'
128 * Owned by the initiator thread.
133 * After the scan is finished, it will contain a pointer to the
134 * top-level directory entry in the directory tree built by the
137 struct GNUNET_FS_ShareTreeItem *toplevel;
140 * Expanded filename (as given by the scan initiator).
141 * The scanner thread stores a copy here, and frees it when it finishes.
143 char *filename_expanded;
146 * A pipe end to read signals from.
147 * Owned by the initiator thread.
149 const struct GNUNET_DISK_FileHandle *stop_read;
152 * 1 if the scanner should stop, 0 otherwise. Set in response
153 * to communication errors or when the initiator wants the scanning
159 * Handle of the pipe end into which the progress messages are written
160 * The pipe is owned by the initiator thread, and there's no way to
161 * close this end without having access to the pipe, so it won't
162 * be closed by the scanner thread.
163 * The initiator MUST keep it alive until the scanner thread is finished.
165 const struct GNUNET_DISK_FileHandle *progress_write;
169 * List of libextractor plugins to use for extracting.
170 * Initialized when the scan starts, removed when it finishes.
172 struct EXTRACTOR_PluginList *plugins;
176 * An opaque structure a pointer to which is returned to the
177 * caller to be used to control the scanner.
179 struct GNUNET_FS_DirScanner
182 * A pipe end to read signals from.
183 * Owned by the initiator thread.
185 const struct GNUNET_DISK_FileHandle *stop_write;
188 * A pipe transfer signals to the scanner.
189 * Owned by the initiator thread.
191 struct GNUNET_DISK_PipeHandle *stop_pipe;
194 * A thread object for the scanner thread.
195 * Owned by the initiator thread.
204 * A task for reading progress messages from the scanner.
206 GNUNET_SCHEDULER_TaskIdentifier progress_read_task;
209 * The end of the pipe that is used to read progress messages.
211 const struct GNUNET_DISK_FileHandle *progress_read;
214 * The pipe that is used to read progress messages.
215 * Owned (along with both of its ends) by the initiator thread.
216 * Only closed after the scanner thread is finished.
218 struct GNUNET_DISK_PipeHandle *progress_pipe;
221 * The function that will be called every time there's a progress
224 GNUNET_FS_DirScannerProgressCallback progress_callback;
227 * A closure for progress_callback.
232 * A pointer to the context of the scanner.
233 * Owned by the initiator thread.
234 * Initiator thread shouldn't touch it until the scanner thread
237 struct AddDirContext *adc;
241 * A structure that forms a singly-linked list that serves as a stack
242 * for metadata-processing function.
244 struct ProcessMetadataStackItem
247 * A pointer to metadata-processing context.
248 * The same in every stack item.
250 struct GNUNET_FS_ProcessMetadataContext *ctx;
253 * This is a singly-linked list. A pointer to its end is kept, and
254 * this pointer is used to walk it backwards.
256 struct ProcessMetadataStackItem *parent;
259 * Map from the hash over the keyword to an 'struct KeywordCounter *'
260 * counter that says how often this keyword was
261 * encountered in the current directory.
263 struct GNUNET_CONTAINER_MultiHashMap *keywordcounter;
266 * Map from the hash over the metadata to an 'struct MetaCounter *'
267 * counter that says how often this metadata was
268 * encountered in the current directory.
270 struct GNUNET_CONTAINER_MultiHashMap *metacounter;
273 * Number of files in the current directory.
275 unsigned int dir_entry_count;
278 * Keywords to exclude from using for KSK since they'll be associated
279 * with the parent as well. NULL for nothing blocked.
281 struct GNUNET_FS_Uri *exclude_ksk;
284 * A share tree item that is being processed.
286 struct GNUNET_FS_ShareTreeItem *item;
289 * Set to GNUNET_YES to indicate that the directory pointer by 'item'
290 * was processed, and we should move on to the next.
291 * Otherwise the directory will be recursed into.
298 * The structure to keep the state of metadata processing
300 struct GNUNET_FS_ProcessMetadataContext
303 * The top of the stack.
305 struct ProcessMetadataStackItem *stack;
308 * Callback to invoke when processing is finished
310 GNUNET_SCHEDULER_Task cb;
318 * Toplevel directory item of the tree to process.
320 struct GNUNET_FS_ShareTreeItem *toplevel;
324 * Called every now and then by the scanner.
325 * Checks the synchronization privitive.
326 * Returns 1 if the scanner should stop, 0 otherwise.
329 should_stop (struct AddDirContext *adc)
333 if (GNUNET_DISK_file_read_non_blocking (adc->stop_read, &c, 1) == 1
342 * Write progress message.
344 * <reason><filename length><filename><directory flag>
345 * If filename is NULL, filename is not written, and its length
346 * is written as 0, and nothing else is written. It signals the initiator
347 * thread that the scanner is finished, and that it can now join its thread.
349 * Also checks if the initiator thread wants the scanner to stop,
350 * Returns 1 to stop scanning (if the signal was received, or
351 * if the pipe was broken somehow), 0 otherwise.
354 write_progress (struct AddDirContext *adc, const char *filename,
355 char is_directory, enum GNUNET_FS_DirScannerProgressUpdateReason reason)
360 if ((adc->do_stop || should_stop (adc)) && reason != GNUNET_DIR_SCANNER_ASKED_TO_STOP
361 && reason != GNUNET_DIR_SCANNER_FINISHED)
365 while ((wr > 0 || errno == EAGAIN) && total_write < sizeof (reason))
367 wr = GNUNET_DISK_file_write_blocking (adc->progress_write,
368 &((char *)&reason)[total_write], sizeof (reason) - total_write);
372 if (sizeof (reason) != total_write)
373 return adc->do_stop = 1;
375 filename_len = strlen (filename) + 1;
380 while ((wr > 0 || errno == EAGAIN) && total_write < sizeof (size_t))
382 wr = GNUNET_DISK_file_write_blocking (adc->progress_write,
383 &((char *)&filename_len)[total_write], sizeof (size_t) - total_write);
387 if (sizeof (size_t) != total_write)
388 return adc->do_stop = 1;
393 while ((wr > 0 || errno == EAGAIN) && total_write < filename_len)
395 wr = GNUNET_DISK_file_write_blocking (adc->progress_write,
396 &((char *)filename)[total_write], filename_len - total_write);
400 if (filename_len != total_write)
401 return adc->do_stop = 1;
404 while ((wr > 0 || errno == EAGAIN) && total_write < sizeof (char))
406 wr = GNUNET_DISK_file_write_blocking (adc->progress_write,
407 &((char *)&is_directory)[total_write], sizeof (char) - total_write);
411 if (sizeof (char) != total_write)
412 return adc->do_stop = 1;
418 * Add the given keyword to the
419 * keyword statistics tracker.
421 * @param cls closure (user-defined)
422 * @param keyword the keyword to count
423 * @param is_mandatory ignored
424 * @return always GNUNET_OK
427 add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory)
429 struct GNUNET_CONTAINER_MultiHashMap *mcm = cls;
430 struct KeywordCounter *cnt, *first_cnt;
434 klen = strlen (keyword) + 1;
435 GNUNET_CRYPTO_hash (keyword, klen - 1, &hc);
436 /* Since the map might contain multiple values per keyword, we only
437 * store one value, and attach all other to it, forming a linked list.
438 * Somewhat easier than retrieving multiple items via callback.
440 first_cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc);
441 for (cnt = first_cnt; cnt && strcmp (cnt->value, keyword) != 0; cnt = cnt->next);
444 cnt = GNUNET_malloc (sizeof (struct KeywordCounter) + klen);
445 cnt->value = (const char *) &cnt[1];
446 memcpy (&cnt[1], keyword, klen);
447 if (first_cnt != NULL)
449 if (first_cnt->prev != NULL)
451 first_cnt->prev->next = cnt;
452 cnt->prev = first_cnt->prev;
454 first_cnt->prev = cnt;
455 cnt->next = first_cnt;
458 GNUNET_CONTAINER_multihashmap_put (mcm, &hc, cnt,
459 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
466 * Type of a function that libextractor calls for each
467 * meta data item found.
469 * @param cls the container multihashmap to update
470 * @param plugin_name name of the plugin that produced this value;
471 * special values can be used (i.e. '<zlib>' for zlib being
472 * used in the main libextractor library and yielding
474 * @param type libextractor-type describing the meta data
475 * @param format basic format information about data
476 * @param data_mime_type mime-type of data (not of the original file);
477 * can be NULL (if mime-type is not known)
478 * @param data actual meta-data found
479 * @param data_len number of bytes in data
480 * @return GNUNET_OK to continue extracting / iterating
483 add_to_meta_counter (void *cls, const char *plugin_name,
484 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
485 const char *data_mime_type, const char *data, size_t data_len)
487 struct GNUNET_CONTAINER_MultiHashMap *map = cls;
489 struct MetaCounter *cnt, *first_cnt;
491 GNUNET_CRYPTO_hash (data, data_len, &key);
492 first_cnt = GNUNET_CONTAINER_multihashmap_get (map, &key);
493 for (cnt = first_cnt; cnt
494 && cnt->data_size != data_len
495 && memcmp (cnt->data, data, cnt->data_size) != 0; cnt = cnt->next);
498 cnt = GNUNET_malloc (sizeof (struct MetaCounter));
500 cnt->data_size = data_len;
501 cnt->plugin_name = plugin_name;
503 cnt->format = format;
504 cnt->data_mime_type = data_mime_type;
506 if (first_cnt != NULL)
508 if (first_cnt->prev != NULL)
510 first_cnt->prev->next = cnt;
511 cnt->prev = first_cnt->prev;
513 first_cnt->prev = cnt;
514 cnt->next = first_cnt;
517 GNUNET_CONTAINER_multihashmap_put (map, &key, cnt,
518 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
525 * Allocates a struct GNUNET_FS_ShareTreeItem and adds it to its parent.
527 static struct GNUNET_FS_ShareTreeItem *
528 make_item (struct GNUNET_FS_ShareTreeItem *parent)
530 struct GNUNET_FS_ShareTreeItem *item;
531 item = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem));
533 item->parent = parent;
535 GNUNET_CONTAINER_DLL_insert (parent->children_head, parent->children_tail,
541 * Extract metadata from a file and add it to the share tree
543 * @param adc context to modify
544 * @param filename name of the file to process
547 extract_file (struct AddDirStack *ads, const char *filename)
549 struct GNUNET_FS_ShareTreeItem *item;
550 const char *short_fn;
552 item = make_item (ads->parent);
554 GNUNET_DISK_file_size (filename, &item->file_size, GNUNET_YES);
555 item->is_directory = GNUNET_NO;
557 item->meta = GNUNET_CONTAINER_meta_data_create ();
558 GNUNET_FS_meta_data_extract_from_file (item->meta, filename,
560 GNUNET_CONTAINER_meta_data_delete (item->meta, EXTRACTOR_METATYPE_FILENAME,
562 short_fn = GNUNET_STRINGS_get_short_name (filename);
564 item->filename = GNUNET_strdup (filename);
565 item->short_filename = GNUNET_strdup (short_fn);
567 GNUNET_CONTAINER_meta_data_insert (item->meta, "<libgnunetfs>",
568 EXTRACTOR_METATYPE_FILENAME,
569 EXTRACTOR_METAFORMAT_UTF8, "text/plain",
570 short_fn, strlen (short_fn) + 1);
571 if (ads->parent == NULL)
573 /* we're finished with the scan, make sure caller gets the top-level
576 ads->adc->toplevel = item;
581 * Remove the keyword from the ksk URI.
583 * @param cls the ksk uri
584 * @param keyword the word to remove
585 * @param is_mandatory ignored
586 * @return always GNUNET_OK
589 remove_keyword (void *cls, const char *keyword, int is_mandatory)
591 struct GNUNET_FS_Uri *ksk = cls;
593 GNUNET_FS_uri_ksk_remove_keyword (ksk, keyword);
598 * Remove keywords from current directory's children, if they are
599 * in the exluded keywords list of that directory.
601 * @param cls the ksk uri
602 * @param keyword the word to remove
603 * @param is_mandatory ignored
604 * @return always GNUNET_OK
607 remove_keywords (struct ProcessMetadataStackItem *stack, struct GNUNET_FS_ShareTreeItem *dir)
609 struct GNUNET_FS_ShareTreeItem *item;
611 for (item = dir->children_head; item; item = item->next)
613 if (stack->exclude_ksk != NULL)
614 GNUNET_FS_uri_ksk_get_keywords (stack->exclude_ksk, &remove_keyword, item->ksk_uri);
620 * Context passed to 'migrate_and_drop'.
622 struct KeywordProcessContext
625 * All the keywords we migrated to the parent.
627 struct GNUNET_FS_Uri *ksk;
630 * How often does a keyword have to occur to be
631 * migrated to the parent?
633 unsigned int threshold;
637 * Context passed to 'migrate_and_drop'.
639 struct MetaProcessContext
642 * All the metadata we copy to the parent.
644 struct GNUNET_CONTAINER_MetaData *meta;
647 * How often does a metadata have to occur to be
648 * migrated to the parent?
650 unsigned int threshold;
655 * Move "frequent" keywords over to the
656 * target ksk uri, free the counters.
660 migrate_and_drop (void *cls, const GNUNET_HashCode * key, void *value)
662 struct KeywordProcessContext *kpc = cls;
663 struct KeywordCounter *counter = value;
665 if (counter->count >= kpc->threshold && counter->count > 1)
667 GNUNET_FS_uri_ksk_add_keyword (kpc->ksk, counter->value, GNUNET_NO);
669 GNUNET_free (counter);
673 * Copy "frequent" metadata items over to the
674 * target metadata container, free the counters.
678 migrate_and_drop_metadata (void *cls, const GNUNET_HashCode * key, void *value)
680 struct MetaProcessContext *mpc = cls;
681 struct MetaCounter *counter = value;
683 if (counter->count >= mpc->threshold && counter->count > 1)
685 GNUNET_CONTAINER_meta_data_insert (mpc->meta,
686 counter->plugin_name,
689 counter->data_mime_type, counter->data,
692 GNUNET_free (counter);
697 * Go over the collected keywords from all entries in the
698 * directory and push common keywords up one level (by
699 * adding it to the returned struct). Do the same for metadata.
700 * Destroys keywordcounter and metacoutner for current directory.
702 * @param adc collection of child meta data
703 * @param exclude_ksk pointer to where moveable keywords will be stored
704 * @param copy_meta pointer to where copyable metadata will be stored
707 process_keywords_and_metadata (struct ProcessMetadataStackItem *stack,
708 struct GNUNET_FS_Uri **exclude_ksk,
709 struct GNUNET_CONTAINER_MetaData **copy_meta)
711 struct KeywordProcessContext kpc;
712 struct MetaProcessContext mpc;
713 struct GNUNET_CONTAINER_MetaData *tmp;
715 /* Surprisingly, it's impossible to create a ksk with 0 keywords directly.
716 * But we can create one from an empty metadata set
718 tmp = GNUNET_CONTAINER_meta_data_create ();
719 kpc.ksk = GNUNET_FS_uri_ksk_create_from_meta_data (tmp);
720 GNUNET_CONTAINER_meta_data_destroy (tmp);
721 mpc.meta = GNUNET_CONTAINER_meta_data_create ();
723 kpc.threshold = mpc.threshold = (stack->dir_entry_count + 1) / 2; /* 50% */
725 GNUNET_CONTAINER_multihashmap_iterate (stack->keywordcounter,
726 &migrate_and_drop, &kpc);
727 GNUNET_CONTAINER_multihashmap_iterate (stack->metacounter,
728 &migrate_and_drop_metadata, &mpc);
730 GNUNET_CONTAINER_multihashmap_destroy (stack->keywordcounter);
731 GNUNET_CONTAINER_multihashmap_destroy (stack->metacounter);
732 *exclude_ksk = kpc.ksk;
733 *copy_meta = mpc.meta;
737 * Function called by the directory iterator to
738 * (recursively) add all of the files in the
739 * directory to the tree.
740 * Called by the directory scanner to initiate the
742 * TODO: find a way to make it non-recursive.
744 * @param cls the 'struct AddDirStack *' we're in
745 * @param filename file or directory to scan
748 scan_directory (void *cls, const char *filename)
750 struct AddDirStack *ads = cls, recurse_ads;
751 struct AddDirContext *adc = ads->adc;
753 struct GNUNET_FS_ShareTreeItem *item;
754 const char *short_fn;
759 return GNUNET_SYSERR;
761 /* If the file doesn't exist (or is not statable for any other reason,
762 * skip it, and report it.
764 if (0 != STAT (filename, &sbuf))
766 do_stop = write_progress (adc, filename, S_ISDIR (sbuf.st_mode),
767 GNUNET_DIR_SCANNER_DOES_NOT_EXIST);
771 /* Report the progress */
772 do_stop = write_progress (adc, filename, S_ISDIR (sbuf.st_mode),
773 GNUNET_DIR_SCANNER_NEW_FILE);
776 /* We were asked to stop, acknowledge that and return */
777 do_stop = write_progress (adc, filename, S_ISDIR (sbuf.st_mode),
778 GNUNET_DIR_SCANNER_ASKED_TO_STOP);
779 return GNUNET_SYSERR;
782 if (!S_ISDIR (sbuf.st_mode))
783 extract_file (ads, filename);
786 item = make_item (ads->parent);
787 item->meta = GNUNET_CONTAINER_meta_data_create ();
789 item->is_directory = GNUNET_YES;
791 recurse_ads.adc = adc;
792 recurse_ads.parent = item;
794 /* recurse into directory */
795 GNUNET_DISK_directory_scan (filename, &scan_directory, &recurse_ads);
797 short_fn = GNUNET_STRINGS_get_short_name (filename);
799 item->filename = GNUNET_strdup (filename);
800 item->short_filename = GNUNET_strdup (short_fn);
802 if (ads->parent == NULL)
804 /* we're finished with the scan, make sure caller gets the top-level
807 adc->toplevel = item;
814 * Signals the scanner to finish the scan as fast as possible.
816 * Can close the pipe if asked to, but that is only used by the
817 * internal call to this function during cleanup. The client
818 * must understand the consequences of closing the pipe too early.
820 * @param ds directory scanner structure
821 * @param close_pipe GNUNET_YES to close
824 GNUNET_FS_directory_scan_finish (struct GNUNET_FS_DirScanner *ds,
828 GNUNET_DISK_file_write (ds->stop_write, &c, 1);
832 if (ds->progress_read_task != GNUNET_SCHEDULER_NO_TASK)
834 GNUNET_SCHEDULER_cancel (ds->progress_read_task);
835 ds->progress_read_task = GNUNET_SCHEDULER_NO_TASK;
837 GNUNET_DISK_pipe_close_end (ds->progress_pipe, GNUNET_DISK_PIPE_END_READ);
838 ds->progress_read = NULL;
843 * Signals the scanner thread to finish (in case it isn't finishing
844 * already) and joins the scanner thread. Closes the pipes, frees the
845 * scanner contexts (both of them), returns the results of the scan.
846 * Results are valid (and have to be freed) even if the scanner had
847 * an error or was rushed to finish prematurely.
848 * Blocks until the scanner is finished.
850 * @param ds directory scanner structure
851 * @return the results of the scan (a directory tree)
853 struct GNUNET_FS_ShareTreeItem *
854 GNUNET_FS_directory_scan_cleanup (struct GNUNET_FS_DirScanner *ds)
856 struct GNUNET_FS_ShareTreeItem *result;
858 GNUNET_FS_directory_scan_finish (ds, GNUNET_YES);
860 WaitForSingleObject (ds->thread, INFINITE);
861 CloseHandle (ds->thread);
863 pthread_join (ds->thread, NULL);
864 pthread_detach (ds->thread);
867 GNUNET_DISK_pipe_close (ds->stop_pipe);
868 GNUNET_DISK_pipe_close (ds->progress_pipe);
869 result = ds->adc->toplevel;
870 GNUNET_free (ds->adc);
876 * The function from which the scanner thread starts
883 run_directory_scan_thread (void *cls)
885 struct AddDirContext *adc = cls;
886 struct AddDirStack ads;
889 scan_directory (&ads, adc->filename_expanded);
890 GNUNET_free (adc->filename_expanded);
891 if (adc->plugins != NULL)
892 EXTRACTOR_plugin_remove_all (adc->plugins);
893 /* Tell the initiator that we're finished, it can now join the thread */
894 write_progress (adc, NULL, 0, GNUNET_DIR_SCANNER_FINISHED);
899 * Called every time there is data to read from the scanner.
900 * Calls the scanner progress handler.
902 * @param cls the closure (directory scanner object)
903 * @param tc task context in which the task is running
906 read_progress_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
908 struct GNUNET_FS_DirScanner *ds;
910 enum GNUNET_FS_DirScannerProgressUpdateReason reason;
920 ds->progress_read_task = GNUNET_SCHEDULER_NO_TASK;
922 if (!(tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
924 ds->progress_callback (ds->cls, ds, NULL, 0, GNUNET_DIR_SCANNER_SHUTDOWN);
928 /* Read one message. If message is malformed or can't be read, end the scanner */
929 total_read = rd = GNUNET_DISK_file_read (ds->progress_read, &reason, sizeof (reason));
930 while (rd > 0 && total_read < sizeof (reason))
932 rd = GNUNET_DISK_file_read (ds->progress_read,
933 &((char *) &reason)[total_read],
934 sizeof (reason) - total_read);
938 if (total_read != sizeof (reason)
939 || reason <= GNUNET_DIR_SCANNER_FIRST
940 || reason >= GNUNET_DIR_SCANNER_LAST)
943 reason = GNUNET_DIR_SCANNER_PROTOCOL_ERROR;
948 total_read = rd = GNUNET_DISK_file_read (ds->progress_read, &filename_len,
950 while (rd > 0 && total_read < sizeof (size_t))
952 rd = GNUNET_DISK_file_read (ds->progress_read,
953 &((char *) &filename_len)[total_read],
954 sizeof (size_t) - total_read);
958 if (rd != sizeof (size_t))
961 reason = GNUNET_DIR_SCANNER_PROTOCOL_ERROR;
966 if (filename_len == 0)
968 else if (filename_len > PATH_MAX)
971 reason = GNUNET_DIR_SCANNER_PROTOCOL_ERROR;
976 filename = GNUNET_malloc (filename_len);
977 total_read = rd = GNUNET_DISK_file_read (ds->progress_read, filename,
979 while (rd > 0 && total_read < filename_len)
981 rd = GNUNET_DISK_file_read (ds->progress_read, &filename[total_read],
982 filename_len - total_read);
986 if (rd != filename_len)
988 GNUNET_free (filename);
989 reason = GNUNET_DIR_SCANNER_PROTOCOL_ERROR;
993 if (!end_it && filename_len > 0)
995 total_read = rd = GNUNET_DISK_file_read (ds->progress_read, &is_directory,
997 while (rd > 0 && total_read < sizeof (char))
999 rd = GNUNET_DISK_file_read (ds->progress_read, &(&is_directory)[total_read],
1000 sizeof (char) - total_read);
1004 if (rd != sizeof (char))
1006 GNUNET_free (filename);
1007 reason = GNUNET_DIR_SCANNER_PROTOCOL_ERROR;
1013 end_it = ds->progress_callback (ds->cls, ds, (const char *) filename, is_directory, reason);
1014 GNUNET_free (filename);
1017 ds->progress_read_task = GNUNET_SCHEDULER_add_read_file (
1018 GNUNET_TIME_UNIT_FOREVER_REL, ds->progress_read, &read_progress_task,
1024 ds->progress_callback (ds->cls, ds, NULL, 0, reason);
1030 * Start a directory scanner thread.
1032 * @param filename name of the directory to scan
1033 * @param GNUNET_YES to not to run libextractor on files (only build a tree)
1034 * @param ex if not NULL, must be a list of extra plugins for extractor
1035 * @param cb the callback to call when there are scanning progress messages
1036 * @param cls closure for 'cb'
1037 * @return directory scanner object to be used for controlling the scanner
1039 struct GNUNET_FS_DirScanner *
1040 GNUNET_FS_directory_scan_start (const char *filename,
1041 int disable_extractor, const char *ex,
1042 GNUNET_FS_DirScannerProgressCallback cb, void *cls)
1045 struct AddDirContext *adc;
1046 char *filename_expanded;
1047 struct GNUNET_FS_DirScanner *ds;
1048 struct GNUNET_DISK_PipeHandle *progress_pipe;
1051 if (0 != STAT (filename, &sbuf))
1054 /* scan_directory() is guaranteed to be given expanded filenames,
1055 * so expand we will!
1057 filename_expanded = GNUNET_STRINGS_filename_expand (filename);
1058 if (filename_expanded == NULL)
1061 progress_pipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
1062 if (progress_pipe == NULL)
1064 GNUNET_free (filename_expanded);
1068 adc = GNUNET_malloc (sizeof (struct AddDirContext));
1070 ds = GNUNET_malloc (sizeof (struct GNUNET_FS_DirScanner));
1074 ds->stop_pipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
1075 if (ds->stop_pipe == NULL)
1079 GNUNET_free (filename_expanded);
1080 GNUNET_DISK_pipe_close (progress_pipe);
1083 ds->stop_write = GNUNET_DISK_pipe_handle (ds->stop_pipe,
1084 GNUNET_DISK_PIPE_END_WRITE);
1085 adc->stop_read = GNUNET_DISK_pipe_handle (ds->stop_pipe,
1086 GNUNET_DISK_PIPE_END_READ);
1088 adc->plugins = NULL;
1089 if (!disable_extractor)
1091 adc->plugins = EXTRACTOR_plugin_add_defaults (
1092 EXTRACTOR_OPTION_DEFAULT_POLICY);
1093 if (ex && strlen (ex) > 0)
1094 adc->plugins = EXTRACTOR_plugin_add_config (adc->plugins, ex,
1095 EXTRACTOR_OPTION_DEFAULT_POLICY);
1098 adc->filename_expanded = filename_expanded;
1099 adc->progress_write = GNUNET_DISK_pipe_handle (progress_pipe,
1100 GNUNET_DISK_PIPE_END_WRITE);
1103 ds->progress_read = GNUNET_DISK_pipe_handle (progress_pipe,
1104 GNUNET_DISK_PIPE_END_READ);
1107 ds->thread = CreateThread (NULL, 0,
1108 (LPTHREAD_START_ROUTINE) &run_directory_scan_thread, (LPVOID) adc,
1110 ok = ds->thread != NULL;
1112 ok = !pthread_create (&ds->thread, NULL, &run_directory_scan_thread,
1118 GNUNET_free (filename_expanded);
1119 GNUNET_DISK_pipe_close (progress_pipe);
1124 ds->progress_callback = cb;
1127 ds->progress_pipe = progress_pipe;
1129 ds->progress_read_task = GNUNET_SCHEDULER_add_read_file (
1130 GNUNET_TIME_UNIT_FOREVER_REL, ds->progress_read, &read_progress_task,
1137 * Task that post-processes the share item tree.
1138 * This processing has to be done in the main thread, because
1139 * it requires access to libgcrypt's hashing functions, and
1140 * libgcrypt is not thread-safe without some special magic.
1142 * @param cls top of the stack
1143 * @param tc task context
1146 trim_share_tree_task (void *cls,
1147 const struct GNUNET_SCHEDULER_TaskContext *tc)
1149 struct ProcessMetadataStackItem *stack = cls;
1150 struct ProcessMetadataStackItem *next = stack;
1151 /* FIXME: figure out what to do when tc says we're shutting down */
1153 /* item == NULL means that we've just finished going over the children of
1154 * current directory.
1156 if (stack->item == NULL)
1158 if (stack->parent->item != NULL)
1160 /* end of a directory */
1161 struct GNUNET_FS_Uri *ksk;
1163 /* use keyword and metadata counters to create lists of keywords to move
1164 * and metadata to copy.
1166 process_keywords_and_metadata (stack, &stack->parent->exclude_ksk, &stack->parent->item->meta);
1168 /* create keywords from metadata (copies all text-metadata as keywords,
1169 * AND parses the directory name we've just added, producing even more
1171 * then merge these keywords with the ones moved from children.
1173 ksk = GNUNET_FS_uri_ksk_create_from_meta_data (stack->parent->item->meta);
1174 stack->parent->item->ksk_uri = GNUNET_FS_uri_ksk_merge (ksk, stack->parent->exclude_ksk);
1175 GNUNET_FS_uri_destroy (ksk);
1177 /* remove moved keywords from children (complete the move) */
1178 remove_keywords (stack->parent, stack->parent->item);
1179 GNUNET_FS_uri_destroy (stack->parent->exclude_ksk);
1181 /* go up the stack */
1182 next = stack->parent;
1183 GNUNET_free (stack);
1184 next->end_directory = GNUNET_YES;
1188 /* we've just finished processing the toplevel directory */
1189 struct GNUNET_FS_ProcessMetadataContext *ctx = stack->ctx;
1191 GNUNET_SCHEDULER_add_continuation (ctx->cb, ctx->cls,
1192 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1193 GNUNET_free (stack->parent);
1194 GNUNET_free (stack);
1198 else if (stack->item->is_directory
1199 && !stack->end_directory
1200 && stack->item->children_head != NULL)
1202 /* recurse into subdirectory */
1203 next = GNUNET_malloc (sizeof (struct ProcessMetadataStackItem));
1204 next->ctx = stack->ctx;
1205 next->item = stack->item->children_head;
1206 next->keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024);
1207 next->metacounter = GNUNET_CONTAINER_multihashmap_create (1024);
1208 next->dir_entry_count = 0;
1209 next->parent = stack;
1213 /* process a child entry (a file or a directory) and move to the next one*/
1214 if (stack->item->is_directory)
1215 stack->end_directory = GNUNET_NO;
1216 if (stack->ctx->toplevel->is_directory)
1218 stack->dir_entry_count++;
1219 GNUNET_CONTAINER_meta_data_iterate (stack->item->meta, &add_to_meta_counter, stack->metacounter);
1221 if (stack->item->is_directory)
1223 char *user = getenv ("USER");
1224 if ((user == NULL) || (0 != strncasecmp (user, stack->item->short_filename, strlen(user))))
1226 /* only use filename if it doesn't match $USER */
1227 GNUNET_CONTAINER_meta_data_insert (stack->item->meta, "<libgnunetfs>",
1228 EXTRACTOR_METATYPE_FILENAME,
1229 EXTRACTOR_METAFORMAT_UTF8,
1230 "text/plain", stack->item->short_filename,
1231 strlen (stack->item->short_filename) + 1);
1232 GNUNET_CONTAINER_meta_data_insert (stack->item->meta, "<libgnunetfs>",
1233 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
1234 EXTRACTOR_METAFORMAT_UTF8,
1235 "text/plain", stack->item->short_filename,
1236 strlen (stack->item->short_filename) + 1);
1240 stack->item->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (stack->item->meta);
1241 if (stack->ctx->toplevel->is_directory)
1243 GNUNET_FS_uri_ksk_get_keywords (stack->item->ksk_uri, &add_to_keyword_counter, stack->keywordcounter);
1245 stack->item = stack->item->next;
1247 /* Call this task again later, if there are more entries to process */
1249 GNUNET_SCHEDULER_add_continuation (&trim_share_tree_task, next,
1250 GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1254 * Process a share item tree, moving frequent keywords up and
1255 * copying frequent metadata up.
1257 * @param toplevel toplevel directory in the tree, returned by the scanner
1258 * @param cb called after processing is done
1259 * @param cls closure for 'cb'
1261 struct GNUNET_FS_ProcessMetadataContext *
1262 GNUNET_FS_trim_share_tree (struct GNUNET_FS_ShareTreeItem *toplevel,
1263 GNUNET_SCHEDULER_Task cb, void *cls)
1265 struct GNUNET_FS_ProcessMetadataContext *ret;
1267 if (toplevel == NULL)
1269 struct GNUNET_SCHEDULER_TaskContext tc;
1270 tc.reason = GNUNET_SCHEDULER_REASON_PREREQ_DONE;
1275 ret = GNUNET_malloc (sizeof (struct GNUNET_FS_ProcessMetadataContext));
1276 ret->toplevel = toplevel;
1277 ret->stack = GNUNET_malloc (sizeof (struct ProcessMetadataStackItem));
1278 ret->stack->ctx = ret;
1279 ret->stack->item = toplevel;
1281 if (ret->stack->ctx->toplevel->is_directory)
1283 ret->stack->keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024);
1284 ret->stack->metacounter = GNUNET_CONTAINER_multihashmap_create (1024);
1287 ret->stack->dir_entry_count = 0;
1288 ret->stack->end_directory = GNUNET_NO;
1290 /* dummy stack entry that tells us we're at the top of the stack */
1291 ret->stack->parent = GNUNET_malloc (sizeof (struct ProcessMetadataStackItem));
1292 ret->stack->parent->ctx = ret;
1297 GNUNET_SCHEDULER_add_continuation (&trim_share_tree_task, ret->stack,
1298 GNUNET_SCHEDULER_REASON_PREREQ_DONE);