2 This file is part of GNUnet.
3 (C) 2001--2012 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/gnunet-auto-share.c
22 * @brief automatically publish files on GNUnet
23 * @author Christian Grothoff
26 * - support loading meta data / keywords from resource file
27 * - add stability timer (a la buildbot)
30 #include "gnunet_util_lib.h"
34 * Item in our work queue (or in the set of files/directories
35 * we have successfully published).
41 * PENDING Work is kept in a linked list.
43 struct WorkItem *prev;
46 * PENDING Work is kept in a linked list.
48 struct WorkItem *next;
51 * Filename of the work item.
56 * Unique identity for this work item (used to detect
57 * if we need to do the work again).
59 struct GNUNET_HashCode id;
64 * Global return value from 'main'.
69 * Are we running 'verbosely'?
74 * Configuration to use.
76 static const struct GNUNET_CONFIGURATION_Handle *cfg;
79 * Name of the configuration file.
81 static char *cfg_filename;
84 * Disable extractor option to use for publishing.
86 static int disable_extractor;
89 * Disable creation time option to use for publishing.
91 static int do_disable_creation_time;
94 * Handle for the 'shutdown' task.
96 static GNUNET_SCHEDULER_TaskIdentifier kill_task;
99 * Handle for the main task that does scanning and working.
101 static GNUNET_SCHEDULER_TaskIdentifier run_task;
104 * Anonymity level option to use for publishing.
106 static unsigned int anonymity_level = 1;
109 * Content priority option to use for publishing.
111 static unsigned int content_priority = 365;
114 * Replication level option to use for publishing.
116 static unsigned int replication_level = 1;
119 * Top-level directory we monitor to auto-publish.
121 static const char *dir_name;
124 * Head of linked list of files still to publish.
126 static struct WorkItem *work_head;
129 * Tail of linked list of files still to publish.
131 static struct WorkItem *work_tail;
134 * Map from the hash of the filename (!) to a 'struct WorkItem'
137 static struct GNUNET_CONTAINER_MultiHashMap *work_finished;
140 * Set to GNUNET_YES if we are shutting down.
142 static int do_shutdown;
145 * Start time of the current round; used to determine how long
146 * one iteration takes (which influences how fast we schedule
149 static struct GNUNET_TIME_Absolute start_time;
152 * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal.
154 static struct GNUNET_DISK_PipeHandle *sigpipe;
157 * Handle to the 'gnunet-publish' process that we executed.
159 static struct GNUNET_OS_Process *publish_proc;
163 * Compute the name of the state database file we will use.
170 GNUNET_asprintf (&ret,
173 (DIR_SEPARATOR == dir_name[strlen(dir_name)-1]) ? "" : DIR_SEPARATOR_STR);
179 * Load the set of 'work_finished' items from disk.
185 struct GNUNET_BIO_ReadHandle *rh;
187 struct GNUNET_HashCode id;
192 fn = get_state_file ();
193 rh = GNUNET_BIO_read_open (fn);
198 if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &n))
203 GNUNET_BIO_read_string (rh, "filename", &fn, 1024)) ||
205 GNUNET_BIO_read (rh, "id", &id, sizeof (struct GNUNET_HashCode))) )
207 wi = GNUNET_malloc (sizeof (struct WorkItem));
211 GNUNET_CRYPTO_hash (wi->filename,
212 strlen (wi->filename),
214 GNUNET_CONTAINER_multihashmap_put (work_finished,
217 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
220 GNUNET_BIO_read_close (rh, &emsg))
224 GNUNET_free_non_null (fn);
226 GNUNET_BIO_read_close (rh, &emsg);
227 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
228 _("Failed to load state: %s\n"),
230 GNUNET_free_non_null (emsg);
235 * Write work item from the work_finished map to the given write handle.
237 * @param cls the 'struct GNUNET_BIO_WriteHandle*'
238 * @param key key of the item in the map (unused)
239 * @param value the 'struct WorkItem' to write
240 * @return GNUNET_OK to continue to iterate (if write worked)
243 write_item (void *cls,
244 const struct GNUNET_HashCode *key,
247 struct GNUNET_BIO_WriteHandle *wh = cls;
248 struct WorkItem *wi = value;
251 GNUNET_BIO_write_string (wh, wi->filename)) ||
253 GNUNET_BIO_write (wh,
255 sizeof (struct GNUNET_HashCode))) )
256 return GNUNET_SYSERR; /* write error, abort iteration */
262 * Save the set of 'work_finished' items on disk.
268 struct GNUNET_BIO_WriteHandle *wh;
271 n = GNUNET_CONTAINER_multihashmap_size (work_finished);
272 fn = get_state_file ();
273 wh = GNUNET_BIO_write_open (fn);
275 GNUNET_BIO_write_int32 (wh, n))
277 (void) GNUNET_BIO_write_close (wh);
278 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
279 _("Failed to save state to file %s\n"),
284 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
287 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
288 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
289 _("Failed to save state to file %s\n"),
296 * Task run on shutdown. Serializes our current state to disk.
298 * @param cls closure, unused
299 * @param tc scheduler context, unused
302 do_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
304 kill_task = GNUNET_SCHEDULER_NO_TASK;
305 do_shutdown = GNUNET_YES;
306 if (NULL != publish_proc)
308 GNUNET_OS_process_kill (publish_proc, SIGKILL);
311 if (GNUNET_SCHEDULER_NO_TASK != run_task)
313 GNUNET_SCHEDULER_cancel (run_task);
314 run_task = GNUNET_SCHEDULER_NO_TASK;
320 * Decide what the next task is (working or scanning) and schedule it.
323 schedule_next_task (void);
327 * Task triggered whenever we receive a SIGCHLD (child
330 * @param cls the 'struct WorkItem' we were working on
334 maint_child_death (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
336 struct WorkItem *wi = cls;
337 struct GNUNET_HashCode key;
339 run_task = GNUNET_SCHEDULER_NO_TASK;
340 GNUNET_break (GNUNET_OK ==
341 GNUNET_OS_process_wait (publish_proc));
342 GNUNET_OS_process_destroy (publish_proc);
344 GNUNET_CRYPTO_hash (wi->filename,
345 strlen (wi->filename),
347 GNUNET_CONTAINER_multihashmap_put (work_finished,
350 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
352 schedule_next_task ();
357 * Signal handler called for SIGCHLD. Triggers the
358 * respective handler by writing to the trigger pipe.
361 sighandler_child_death ()
364 int old_errno = errno; /* back-up errno */
367 GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle
368 (sigpipe, GNUNET_DISK_PIPE_END_WRITE),
370 errno = old_errno; /* restore errno */
375 * Function called to process work items.
377 * @param cls closure, NULL
378 * @param tc scheduler context (unused)
382 const struct GNUNET_SCHEDULER_TaskContext *tc)
384 static char *argv[14];
385 static char anon_level[20];
386 static char content_prio[20];
387 static char repl_level[20];
389 const struct GNUNET_DISK_FileHandle *pr;
392 run_task = GNUNET_SCHEDULER_NO_TASK;
394 GNUNET_CONTAINER_DLL_remove (work_head,
398 argv[argc++] = "gnunet-publish";
401 if (disable_extractor)
403 if (do_disable_creation_time)
406 argv[argc++] = cfg_filename;
407 GNUNET_snprintf (anon_level, sizeof (anon_level),
408 "%u", anonymity_level);
410 argv[argc++] = anon_level;
411 GNUNET_snprintf (content_prio, sizeof (content_prio),
412 "%u", content_priority);
414 argv[argc++] = content_prio;
415 GNUNET_snprintf (repl_level, sizeof (repl_level),
416 "%u", replication_level);
418 argv[argc++] = repl_level;
419 argv[argc++] = wi->filename;
421 publish_proc = GNUNET_OS_start_process_vap (GNUNET_YES,
425 if (NULL == publish_proc)
427 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
428 _("Failed to run `%s'\n"),
430 GNUNET_CONTAINER_DLL_insert (work_head,
433 run_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
438 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
440 GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
441 pr, &maint_child_death, wi);
446 * Recursively scan the given file/directory structure to determine
447 * a unique ID that represents the current state of the hierarchy.
449 * @param cls where to store the unique ID we are computing
450 * @param filename file to scan
451 * @return GNUNET_OK (always)
454 determine_id (void *cls,
455 const char *filename)
457 struct GNUNET_HashCode *id = cls;
459 struct GNUNET_HashCode fx[2];
460 struct GNUNET_HashCode ft;
462 if (NULL != strstr (filename,
463 DIR_SEPARATOR_STR ".auto-share"))
464 return GNUNET_OK; /* skip internal file */
465 if (0 != STAT (filename, &sbuf))
467 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
470 GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]);
471 if (!S_ISDIR (sbuf.st_mode))
473 uint64_t fsize = GNUNET_htonll (sbuf.st_size);
475 GNUNET_CRYPTO_hash (&fsize, sizeof (uint64_t), &fx[1]);
479 memset (&fx[1], 1, sizeof (struct GNUNET_HashCode));
480 GNUNET_DISK_directory_scan (filename,
484 /* use hash here to make hierarchical structure distinct from
485 all files on the same level */
486 GNUNET_CRYPTO_hash (fx, sizeof (fx), &ft);
487 /* use XOR here so that order of the files in the directory
489 GNUNET_CRYPTO_hash_xor (&ft, id, id);
495 * Function called with a filename (or directory name) to publish
496 * (if it has changed since the last time we published it). This function
497 * is called for the top-level files only.
499 * @param cls closure, NULL
500 * @param filename complete filename (absolute path)
501 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR during shutdown
505 const char *filename)
508 struct GNUNET_HashCode key;
509 struct GNUNET_HashCode id;
511 if (GNUNET_YES == do_shutdown)
512 return GNUNET_SYSERR;
513 GNUNET_CRYPTO_hash (filename,
516 wi = GNUNET_CONTAINER_multihashmap_get (work_finished,
518 memset (&id, 0, sizeof (struct GNUNET_HashCode));
519 determine_id (&id, filename);
522 if (0 == memcmp (&id,
524 sizeof (struct GNUNET_HashCode)))
525 return GNUNET_OK; /* skip: we did this one already */
526 /* contents changed, need to re-do the directory... */
527 GNUNET_CONTAINER_multihashmap_remove (work_finished,
534 wi = GNUNET_malloc (sizeof (struct WorkItem));
535 wi->filename = GNUNET_strdup (filename);
537 GNUNET_CONTAINER_DLL_insert (work_head,
540 if (GNUNET_YES == do_shutdown)
541 return GNUNET_SYSERR;
547 * Periodically run task to update our view of the directory to share.
550 * @param tc scheduler context, unused
553 scan (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
555 run_task = GNUNET_SCHEDULER_NO_TASK;
556 start_time = GNUNET_TIME_absolute_get ();
557 (void) GNUNET_DISK_directory_scan (dir_name,
560 schedule_next_task ();
565 * Decide what the next task is (working or scanning) and schedule it.
568 schedule_next_task ()
570 struct GNUNET_TIME_Relative delay;
572 if (GNUNET_YES == do_shutdown)
574 if (NULL == work_head)
576 /* delay by at most 4h, at least 1s, and otherwise in between depending
577 on how long it took to scan */
578 delay = GNUNET_TIME_absolute_get_duration (start_time);
579 delay = GNUNET_TIME_relative_min (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
581 GNUNET_TIME_relative_multiply (delay,
583 delay = GNUNET_TIME_relative_max (delay,
584 GNUNET_TIME_UNIT_MINUTES);
585 run_task = GNUNET_SCHEDULER_add_delayed (delay,
591 run_task = GNUNET_SCHEDULER_add_now (&work, NULL);
597 * Main function that will be run by the scheduler.
600 * @param args remaining command-line arguments
601 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
602 * @param c configuration
605 run (void *cls, char *const *args, const char *cfgfile,
606 const struct GNUNET_CONFIGURATION_Handle *c)
608 /* check arguments */
609 if ((args[0] == NULL) || (args[1] != NULL) ||
610 (GNUNET_YES != GNUNET_DISK_directory_test (args[0])))
612 printf (_("You must specify one and only one directory name for automatic publication.\n"));
616 cfg_filename = GNUNET_strdup (cfgfile);
619 work_finished = GNUNET_CONTAINER_multihashmap_create (1024);
621 run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
625 GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task,
631 * Free memory associated with the work item from the work_finished map.
633 * @param cls NULL (unused)
634 * @param key key of the item in the map (unused)
635 * @param value the 'struct WorkItem' to free
636 * @return GNUNET_OK to continue to iterate
639 free_item (void *cls,
640 const struct GNUNET_HashCode *key,
643 struct WorkItem *wi = value;
645 GNUNET_free (wi->filename);
651 * The main function to automatically publish content to GNUnet.
653 * @param argc number of arguments from the command line
654 * @param argv command line arguments
655 * @return 0 ok, 1 on error
658 main (int argc, char *const *argv)
660 static const struct GNUNET_GETOPT_CommandLineOption options[] = {
661 {'a', "anonymity", "LEVEL",
662 gettext_noop ("set the desired LEVEL of sender-anonymity"),
663 1, &GNUNET_GETOPT_set_uint, &anonymity_level},
664 {'d', "disable-creation-time", NULL,
666 ("disable adding the creation time to the metadata of the uploaded file"),
667 0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
668 {'D', "disable-extractor", NULL,
669 gettext_noop ("do not use libextractor to add keywords or metadata"),
670 0, &GNUNET_GETOPT_set_one, &disable_extractor},
671 {'p', "priority", "PRIORITY",
672 gettext_noop ("specify the priority of the content"),
673 1, &GNUNET_GETOPT_set_uint, &content_priority},
674 {'r', "replication", "LEVEL",
675 gettext_noop ("set the desired replication LEVEL"),
676 1, &GNUNET_GETOPT_set_uint, &replication_level},
677 {'V', "verbose", NULL,
678 gettext_noop ("be verbose (print progress information)"),
679 0, &GNUNET_GETOPT_set_one, &verbose},
680 GNUNET_GETOPT_OPTION_END
684 struct GNUNET_SIGNAL_Context *shc_chld;
686 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
688 sigpipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
689 GNUNET_assert (sigpipe != NULL);
691 GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
693 GNUNET_PROGRAM_run (argc, argv, "gnunet-auto-share [OPTIONS] FILENAME",
695 ("Automatically publish files from a directory on GNUnet"),
696 options, &run, NULL)) ? ret : 1;
697 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
700 GNUNET_CONTAINER_multihashmap_destroy (work_finished);
701 while (NULL != (wi = work_head))
703 GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
704 GNUNET_free (wi->filename);
707 GNUNET_SIGNAL_handler_uninstall (shc_chld);
709 GNUNET_DISK_pipe_close (sigpipe);
711 GNUNET_free (cfg_filename);
716 /* end of gnunet-auto-share.c */