2 This file is part of GNUnet.
3 (C) 2001--2012 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/gnunet-auto-share.c
22 * @brief automatically publish files on GNUnet
23 * @author Christian Grothoff
26 * - support loading meta data / keywords from resource file
27 * - add stability timer (a la buildbot)
30 #include "gnunet_util_lib.h"
32 #define MIN_FREQUENCY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
34 #define MAX_FREQUENCY GNUNET_TIME_UNIT_MINUTES
38 * Item in our work queue (or in the set of files/directories
39 * we have successfully published).
45 * PENDING Work is kept in a linked list.
47 struct WorkItem *prev;
50 * PENDING Work is kept in a linked list.
52 struct WorkItem *next;
55 * Filename of the work item.
60 * Unique identity for this work item (used to detect
61 * if we need to do the work again).
63 struct GNUNET_HashCode id;
68 * Global return value from 'main'.
73 * Are we running 'verbosely'?
78 * Configuration to use.
80 static const struct GNUNET_CONFIGURATION_Handle *cfg;
83 * Name of the configuration file.
85 static char *cfg_filename;
88 * Disable extractor option to use for publishing.
90 static int disable_extractor;
93 * Disable creation time option to use for publishing.
95 static int do_disable_creation_time;
98 * Handle for the 'shutdown' task.
100 static struct GNUNET_SCHEDULER_Task * kill_task;
103 * Handle for the main task that does scanning and working.
105 static struct GNUNET_SCHEDULER_Task * run_task;
108 * Anonymity level option to use for publishing.
110 static unsigned int anonymity_level = 1;
113 * Content priority option to use for publishing.
115 static unsigned int content_priority = 365;
118 * Replication level option to use for publishing.
120 static unsigned int replication_level = 1;
123 * Top-level directory we monitor to auto-publish.
125 static const char *dir_name;
128 * Head of linked list of files still to publish.
130 static struct WorkItem *work_head;
133 * Tail of linked list of files still to publish.
135 static struct WorkItem *work_tail;
138 * Map from the hash of the filename (!) to a 'struct WorkItem'
141 static struct GNUNET_CONTAINER_MultiHashMap *work_finished;
144 * Set to GNUNET_YES if we are shutting down.
146 static int do_shutdown;
149 * Start time of the current round; used to determine how long
150 * one iteration takes (which influences how fast we schedule
153 static struct GNUNET_TIME_Absolute start_time;
156 * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal.
158 static struct GNUNET_DISK_PipeHandle *sigpipe;
161 * Handle to the 'gnunet-publish' process that we executed.
163 static struct GNUNET_OS_Process *publish_proc;
167 * Compute the name of the state database file we will use.
174 GNUNET_asprintf (&ret,
177 (DIR_SEPARATOR == dir_name[strlen(dir_name)-1]) ? "" : DIR_SEPARATOR_STR);
183 * Load the set of 'work_finished' items from disk.
189 struct GNUNET_BIO_ReadHandle *rh;
191 struct GNUNET_HashCode id;
196 fn = get_state_file ();
197 rh = GNUNET_BIO_read_open (fn);
202 if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &n))
207 GNUNET_BIO_read_string (rh, "filename", &fn, 1024)) ||
209 GNUNET_BIO_read (rh, "id", &id, sizeof (struct GNUNET_HashCode))) )
211 wi = GNUNET_new (struct WorkItem);
214 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
215 "Loaded serialization ID for `%s' is `%s'\n",
219 GNUNET_CRYPTO_hash (wi->filename,
220 strlen (wi->filename),
222 GNUNET_break (GNUNET_OK ==
223 GNUNET_CONTAINER_multihashmap_put (work_finished,
226 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
229 GNUNET_BIO_read_close (rh, &emsg))
233 GNUNET_free_non_null (fn);
235 (void) GNUNET_BIO_read_close (rh, &emsg);
236 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
237 _("Failed to load state: %s\n"),
239 GNUNET_free_non_null (emsg);
244 * Write work item from the work_finished map to the given write handle.
246 * @param cls the 'struct GNUNET_BIO_WriteHandle*'
247 * @param key key of the item in the map (unused)
248 * @param value the 'struct WorkItem' to write
249 * @return GNUNET_OK to continue to iterate (if write worked)
252 write_item (void *cls,
253 const struct GNUNET_HashCode *key,
256 struct GNUNET_BIO_WriteHandle *wh = cls;
257 struct WorkItem *wi = value;
259 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
260 "Saving serialization ID of file `%s' with value `%s'\n",
262 GNUNET_h2s (&wi->id));
264 GNUNET_BIO_write_string (wh, wi->filename)) ||
266 GNUNET_BIO_write (wh,
268 sizeof (struct GNUNET_HashCode))) )
269 return GNUNET_SYSERR; /* write error, abort iteration */
275 * Save the set of 'work_finished' items on disk.
281 struct GNUNET_BIO_WriteHandle *wh;
284 n = GNUNET_CONTAINER_multihashmap_size (work_finished);
285 fn = get_state_file ();
286 wh = GNUNET_BIO_write_open (fn);
289 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
290 _("Failed to save state to file %s\n"),
296 GNUNET_BIO_write_int32 (wh, n))
298 (void) GNUNET_BIO_write_close (wh);
299 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
300 _("Failed to save state to file %s\n"),
305 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
308 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
309 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
310 _("Failed to save state to file %s\n"),
317 * Task run on shutdown. Serializes our current state to disk.
319 * @param cls closure, unused
320 * @param tc scheduler context, unused
323 do_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
326 do_shutdown = GNUNET_YES;
327 if (NULL != publish_proc)
329 GNUNET_OS_process_kill (publish_proc, SIGKILL);
332 if (NULL != run_task)
334 GNUNET_SCHEDULER_cancel (run_task);
341 * Decide what the next task is (working or scanning) and schedule it.
344 schedule_next_task (void);
348 * Task triggered whenever we receive a SIGCHLD (child
351 * @param cls the 'struct WorkItem' we were working on
355 maint_child_death (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
357 struct WorkItem *wi = cls;
358 struct GNUNET_HashCode key;
359 enum GNUNET_OS_ProcessStatusType type;
363 const struct GNUNET_DISK_FileHandle *pr;
367 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
368 if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
370 /* shutdown scheduled us, ignore! */
372 GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
373 pr, &maint_child_death, wi);
377 ret = GNUNET_OS_process_status (publish_proc,
380 GNUNET_assert (GNUNET_SYSERR != ret);
381 if (GNUNET_NO == ret)
384 GNUNET_OS_process_kill (publish_proc, SIGKILL);
385 type = GNUNET_OS_PROCESS_SIGNALED;
387 GNUNET_OS_process_destroy (publish_proc);
389 /* consume the signal */
390 GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof (c)));
392 if (GNUNET_YES == do_shutdown)
394 GNUNET_free (wi->filename);
398 if ( (GNUNET_OS_PROCESS_EXITED == type) &&
401 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
402 _("Publication of `%s' done\n"),
404 GNUNET_CRYPTO_hash (wi->filename,
405 strlen (wi->filename),
407 GNUNET_break (GNUNET_OK ==
408 GNUNET_CONTAINER_multihashmap_put (work_finished,
411 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
415 GNUNET_CONTAINER_DLL_insert_tail (work_head,
420 schedule_next_task ();
425 * Signal handler called for SIGCHLD. Triggers the
426 * respective handler by writing to the trigger pipe.
429 sighandler_child_death ()
432 int old_errno = errno; /* back-up errno */
435 GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle
436 (sigpipe, GNUNET_DISK_PIPE_END_WRITE),
438 errno = old_errno; /* restore errno */
443 * Function called to process work items.
445 * @param cls closure, NULL
446 * @param tc scheduler context (unused)
450 const struct GNUNET_SCHEDULER_TaskContext *tc)
452 static char *argv[14];
453 static char anon_level[20];
454 static char content_prio[20];
455 static char repl_level[20];
457 const struct GNUNET_DISK_FileHandle *pr;
462 GNUNET_CONTAINER_DLL_remove (work_head,
466 argv[argc++] = "gnunet-publish";
469 if (disable_extractor)
471 if (do_disable_creation_time)
474 argv[argc++] = cfg_filename;
475 GNUNET_snprintf (anon_level, sizeof (anon_level),
476 "%u", anonymity_level);
478 argv[argc++] = anon_level;
479 GNUNET_snprintf (content_prio, sizeof (content_prio),
480 "%u", content_priority);
482 argv[argc++] = content_prio;
483 GNUNET_snprintf (repl_level, sizeof (repl_level),
484 "%u", replication_level);
486 argv[argc++] = repl_level;
487 argv[argc++] = wi->filename;
489 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
490 _("Publishing `%s'\n"),
492 publish_proc = GNUNET_OS_start_process_vap (GNUNET_YES,
496 if (NULL == publish_proc)
498 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
499 _("Failed to run `%s'\n"),
501 GNUNET_CONTAINER_DLL_insert (work_head,
504 run_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
509 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
511 GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
512 pr, &maint_child_death, wi);
517 * Recursively scan the given file/directory structure to determine
518 * a unique ID that represents the current state of the hierarchy.
520 * @param cls where to store the unique ID we are computing
521 * @param filename file to scan
522 * @return GNUNET_OK (always)
525 determine_id (void *cls,
526 const char *filename)
528 struct GNUNET_HashCode *id = cls;
530 struct GNUNET_HashCode fx[2];
531 struct GNUNET_HashCode ft;
533 if (0 != STAT (filename, &sbuf))
535 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
538 GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]);
539 if (!S_ISDIR (sbuf.st_mode))
543 fattr[0] = GNUNET_htonll (sbuf.st_size);
544 fattr[0] = GNUNET_htonll (sbuf.st_mtime);
546 GNUNET_CRYPTO_hash (fattr, sizeof (fattr), &fx[1]);
550 memset (&fx[1], 1, sizeof (struct GNUNET_HashCode));
551 GNUNET_DISK_directory_scan (filename,
555 /* use hash here to make hierarchical structure distinct from
556 all files on the same level */
557 GNUNET_CRYPTO_hash (fx, sizeof (fx), &ft);
558 /* use XOR here so that order of the files in the directory
560 GNUNET_CRYPTO_hash_xor (&ft, id, id);
566 * Function called with a filename (or directory name) to publish
567 * (if it has changed since the last time we published it). This function
568 * is called for the top-level files only.
570 * @param cls closure, NULL
571 * @param filename complete filename (absolute path)
572 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR during shutdown
576 const char *filename)
579 struct GNUNET_HashCode key;
580 struct GNUNET_HashCode id;
582 if (GNUNET_YES == do_shutdown)
583 return GNUNET_SYSERR;
584 if ( (NULL != strstr (filename,
586 (NULL != strstr (filename,
588 return GNUNET_OK; /* skip internal file */
589 GNUNET_CRYPTO_hash (filename,
592 wi = GNUNET_CONTAINER_multihashmap_get (work_finished,
594 memset (&id, 0, sizeof (struct GNUNET_HashCode));
595 determine_id (&id, filename);
598 if (0 == memcmp (&id,
600 sizeof (struct GNUNET_HashCode)))
601 return GNUNET_OK; /* skip: we did this one already */
602 /* contents changed, need to re-do the directory... */
603 GNUNET_assert (GNUNET_YES ==
604 GNUNET_CONTAINER_multihashmap_remove (work_finished,
610 wi = GNUNET_new (struct WorkItem);
611 wi->filename = GNUNET_strdup (filename);
614 GNUNET_CONTAINER_DLL_insert (work_head,
617 if (GNUNET_YES == do_shutdown)
618 return GNUNET_SYSERR;
624 * Periodically run task to update our view of the directory to share.
627 * @param tc scheduler context, unused
630 scan (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
633 start_time = GNUNET_TIME_absolute_get ();
634 (void) GNUNET_DISK_directory_scan (dir_name,
637 schedule_next_task ();
642 * Decide what the next task is (working or scanning) and schedule it.
645 schedule_next_task ()
647 struct GNUNET_TIME_Relative delay;
649 if (GNUNET_YES == do_shutdown)
651 if (NULL == work_head)
653 /* delay by at most 4h, at least 1s, and otherwise in between depending
654 on how long it took to scan */
655 delay = GNUNET_TIME_absolute_get_duration (start_time);
656 delay = GNUNET_TIME_relative_min (MIN_FREQUENCY,
657 GNUNET_TIME_relative_multiply (delay,
659 delay = GNUNET_TIME_relative_max (delay,
661 run_task = GNUNET_SCHEDULER_add_delayed (delay,
667 run_task = GNUNET_SCHEDULER_add_now (&work, NULL);
673 * Main function that will be run by the scheduler.
676 * @param args remaining command-line arguments
677 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
678 * @param c configuration
681 run (void *cls, char *const *args, const char *cfgfile,
682 const struct GNUNET_CONFIGURATION_Handle *c)
684 /* check arguments */
685 if ((args[0] == NULL) || (args[1] != NULL) ||
686 (GNUNET_YES != GNUNET_DISK_directory_test (args[0], GNUNET_YES)))
688 printf (_("You must specify one and only one directory name for automatic publication.\n"));
692 cfg_filename = GNUNET_strdup (cfgfile);
695 work_finished = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
697 run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
701 GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task,
707 * Free memory associated with the work item from the work_finished map.
709 * @param cls NULL (unused)
710 * @param key key of the item in the map (unused)
711 * @param value the 'struct WorkItem' to free
712 * @return GNUNET_OK to continue to iterate
715 free_item (void *cls,
716 const struct GNUNET_HashCode *key,
719 struct WorkItem *wi = value;
721 GNUNET_free (wi->filename);
728 * The main function to automatically publish content to GNUnet.
730 * @param argc number of arguments from the command line
731 * @param argv command line arguments
732 * @return 0 ok, 1 on error
735 main (int argc, char *const *argv)
737 static const struct GNUNET_GETOPT_CommandLineOption options[] = {
738 {'a', "anonymity", "LEVEL",
739 gettext_noop ("set the desired LEVEL of sender-anonymity"),
740 1, &GNUNET_GETOPT_set_uint, &anonymity_level},
741 {'d', "disable-creation-time", NULL,
743 ("disable adding the creation time to the metadata of the uploaded file"),
744 0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
745 {'D', "disable-extractor", NULL,
746 gettext_noop ("do not use libextractor to add keywords or metadata"),
747 0, &GNUNET_GETOPT_set_one, &disable_extractor},
748 {'p', "priority", "PRIORITY",
749 gettext_noop ("specify the priority of the content"),
750 1, &GNUNET_GETOPT_set_uint, &content_priority},
751 {'r', "replication", "LEVEL",
752 gettext_noop ("set the desired replication LEVEL"),
753 1, &GNUNET_GETOPT_set_uint, &replication_level},
754 {'V', "verbose", NULL,
755 gettext_noop ("be verbose (print progress information)"),
756 0, &GNUNET_GETOPT_set_one, &verbose},
757 GNUNET_GETOPT_OPTION_END
761 struct GNUNET_SIGNAL_Context *shc_chld;
763 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
765 sigpipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
766 GNUNET_assert (sigpipe != NULL);
768 GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
770 GNUNET_PROGRAM_run (argc, argv, "gnunet-auto-share [OPTIONS] FILENAME",
772 ("Automatically publish files from a directory on GNUnet"),
773 options, &run, NULL)) ? ret : 1;
774 if (NULL != work_finished)
776 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
779 GNUNET_CONTAINER_multihashmap_destroy (work_finished);
781 while (NULL != (wi = work_head))
783 GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
784 GNUNET_free (wi->filename);
787 GNUNET_SIGNAL_handler_uninstall (shc_chld);
789 GNUNET_DISK_pipe_close (sigpipe);
791 GNUNET_free (cfg_filename);
793 GNUNET_free ((void*) argv);
797 /* end of gnunet-auto-share.c */