2 This file is part of GNUnet.
3 Copyright (C) 2001--2012 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file fs/gnunet-auto-share.c
22 * @brief automatically publish files on GNUnet
23 * @author Christian Grothoff
26 * - support loading meta data / keywords from resource file
27 * - add stability timer (a la buildbot)
30 #include "gnunet_util_lib.h"
32 #define MAX_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
34 #define MIN_DELAY GNUNET_TIME_UNIT_MINUTES
38 * Item in our work queue (or in the set of files/directories
39 * we have successfully published).
44 * PENDING Work is kept in a linked list.
46 struct WorkItem *prev;
49 * PENDING Work is kept in a linked list.
51 struct WorkItem *next;
54 * Filename of the work item.
59 * Unique identity for this work item (used to detect
60 * if we need to do the work again).
62 struct GNUNET_HashCode id;
67 * Global return value from 'main'.
72 * Are we running 'verbosely'?
74 static unsigned int verbose;
77 * Configuration to use.
79 static const struct GNUNET_CONFIGURATION_Handle *cfg;
82 * Name of the configuration file.
84 static char *cfg_filename;
87 * Disable extractor option to use for publishing.
89 static int disable_extractor;
92 * Disable creation time option to use for publishing.
94 static int do_disable_creation_time;
97 * Handle for the main task that does scanning and working.
99 static struct GNUNET_SCHEDULER_Task *run_task;
102 * Anonymity level option to use for publishing.
104 static unsigned int anonymity_level = 1;
107 * Content priority option to use for publishing.
109 static unsigned int content_priority = 365;
112 * Replication level option to use for publishing.
114 static unsigned int replication_level = 1;
117 * Top-level directory we monitor to auto-publish.
119 static const char *dir_name;
122 * Head of linked list of files still to publish.
124 static struct WorkItem *work_head;
127 * Tail of linked list of files still to publish.
129 static struct WorkItem *work_tail;
132 * Map from the hash of the filename (!) to a `struct WorkItem`
135 static struct GNUNET_CONTAINER_MultiHashMap *work_finished;
138 * Set to #GNUNET_YES if we are shutting down.
140 static int do_shutdown;
143 * Start time of the current round; used to determine how long
144 * one iteration takes (which influences how fast we schedule
147 static struct GNUNET_TIME_Absolute start_time;
150 * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal.
152 static struct GNUNET_DISK_PipeHandle *sigpipe;
155 * Handle to the 'gnunet-publish' process that we executed.
157 static struct GNUNET_OS_Process *publish_proc;
161 * Compute the name of the state database file we will use.
168 GNUNET_asprintf (&ret,
171 (DIR_SEPARATOR == dir_name[strlen (dir_name) - 1])
173 : DIR_SEPARATOR_STR);
179 * Load the set of #work_finished items from disk.
185 struct GNUNET_BIO_ReadHandle *rh;
187 struct GNUNET_HashCode id;
192 fn = get_state_file ();
193 rh = GNUNET_BIO_read_open (fn);
198 if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &n))
202 if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "filename", &fn, 1024)) ||
204 GNUNET_BIO_read (rh, "id", &id, sizeof(struct GNUNET_HashCode))))
206 wi = GNUNET_new (struct WorkItem);
209 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
210 "Loaded serialization ID for `%s' is `%s'\n",
214 GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &id);
215 GNUNET_break (GNUNET_OK ==
216 GNUNET_CONTAINER_multihashmap_put (
220 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
222 if (GNUNET_OK == GNUNET_BIO_read_close (rh, &emsg))
226 GNUNET_free_non_null (fn);
228 (void) GNUNET_BIO_read_close (rh, &emsg);
229 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
230 _ ("Failed to load state: %s\n"),
232 GNUNET_free_non_null (emsg);
237 * Write work item from the #work_finished map to the given write handle.
239 * @param cls the `struct GNUNET_BIO_WriteHandle *`
240 * @param key key of the item in the map (unused)
241 * @param value the `struct WorkItem` to write
242 * @return #GNUNET_OK to continue to iterate (if write worked)
245 write_item (void *cls, const struct GNUNET_HashCode *key, void *value)
247 struct GNUNET_BIO_WriteHandle *wh = cls;
248 struct WorkItem *wi = value;
250 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
251 "Saving serialization ID of file `%s' with value `%s'\n",
253 GNUNET_h2s (&wi->id));
254 if ((GNUNET_OK != GNUNET_BIO_write_string (wh, wi->filename)) ||
256 GNUNET_BIO_write (wh, &wi->id, sizeof(struct GNUNET_HashCode))))
257 return GNUNET_SYSERR; /* write error, abort iteration */
263 * Save the set of #work_finished items on disk.
269 struct GNUNET_BIO_WriteHandle *wh;
272 n = GNUNET_CONTAINER_multihashmap_size (work_finished);
273 fn = get_state_file ();
274 wh = GNUNET_BIO_write_open (fn);
277 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
278 _ ("Failed to save state to file %s\n"),
283 if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, n))
285 (void) GNUNET_BIO_write_close (wh);
286 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
287 _ ("Failed to save state to file %s\n"),
292 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, &write_item, wh);
293 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
294 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
295 _ ("Failed to save state to file %s\n"),
302 * Task run on shutdown. Serializes our current state to disk.
304 * @param cls closure, unused
307 do_stop_task (void *cls)
309 do_shutdown = GNUNET_YES;
310 if (NULL != publish_proc)
312 GNUNET_OS_process_kill (publish_proc, SIGKILL);
315 if (NULL != run_task)
317 GNUNET_SCHEDULER_cancel (run_task);
324 * Decide what the next task is (working or scanning) and schedule it.
327 schedule_next_task (void);
331 * Task triggered whenever we receive a SIGCHLD (child
334 * @param cls the `struct WorkItem` we were working on
337 maint_child_death (void *cls)
339 struct WorkItem *wi = cls;
340 struct GNUNET_HashCode key;
341 enum GNUNET_OS_ProcessStatusType type;
345 const struct GNUNET_DISK_FileHandle *pr;
346 const struct GNUNET_SCHEDULER_TaskContext *tc;
349 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
350 tc = GNUNET_SCHEDULER_get_task_context ();
351 if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
353 /* shutdown scheduled us, someone else will kill child,
354 we should just try again */
355 run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
361 /* consume the signal */
362 GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof(c)));
364 ret = GNUNET_OS_process_status (publish_proc, &type, &code);
365 GNUNET_assert (GNUNET_SYSERR != ret);
366 if (GNUNET_NO == ret)
368 /* process still running? Then where did the SIGCHLD come from?
369 Well, let's declare it spurious (kernel bug?) and keep rolling.
372 run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
378 GNUNET_assert (GNUNET_OK == ret);
380 GNUNET_OS_process_destroy (publish_proc);
383 if (GNUNET_YES == do_shutdown)
385 GNUNET_free (wi->filename);
389 if ((GNUNET_OS_PROCESS_EXITED == type) && (0 == code))
391 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
392 _ ("Publication of `%s' done\n"),
394 GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &key);
395 GNUNET_break (GNUNET_OK ==
396 GNUNET_CONTAINER_multihashmap_put (
400 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
404 GNUNET_CONTAINER_DLL_insert_tail (work_head, work_tail, wi);
407 schedule_next_task ();
412 * Signal handler called for SIGCHLD. Triggers the
413 * respective handler by writing to the trigger pipe.
416 sighandler_child_death ()
419 int old_errno = errno; /* back-up errno */
423 GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (sigpipe,
424 GNUNET_DISK_PIPE_END_WRITE),
427 errno = old_errno; /* restore errno */
432 * Function called to process work items.
434 * @param cls closure, NULL
439 static char *argv[14];
440 static char anon_level[20];
441 static char content_prio[20];
442 static char repl_level[20];
444 const struct GNUNET_DISK_FileHandle *pr;
449 GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
451 argv[argc++] = "gnunet-publish";
454 if (disable_extractor)
456 if (do_disable_creation_time)
459 argv[argc++] = cfg_filename;
460 GNUNET_snprintf (anon_level, sizeof(anon_level), "%u", anonymity_level);
462 argv[argc++] = anon_level;
463 GNUNET_snprintf (content_prio, sizeof(content_prio), "%u", content_priority);
465 argv[argc++] = content_prio;
466 GNUNET_snprintf (repl_level, sizeof(repl_level), "%u", replication_level);
468 argv[argc++] = repl_level;
469 argv[argc++] = wi->filename;
471 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Publishing `%s'\n"), wi->filename);
472 GNUNET_assert (NULL == publish_proc);
473 publish_proc = GNUNET_OS_start_process_vap (GNUNET_YES,
480 if (NULL == publish_proc)
482 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
483 _ ("Failed to run `%s'\n"),
485 GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi);
487 GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &work, NULL);
490 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
491 run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
499 * Recursively scan the given file/directory structure to determine
500 * a unique ID that represents the current state of the hierarchy.
502 * @param cls where to store the unique ID we are computing
503 * @param filename file to scan
504 * @return #GNUNET_OK (always)
507 determine_id (void *cls, const char *filename)
509 struct GNUNET_HashCode *id = cls;
511 struct GNUNET_HashCode fx[2];
512 struct GNUNET_HashCode ft;
514 if (0 != stat (filename, &sbuf))
516 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
519 GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]);
520 if (! S_ISDIR (sbuf.st_mode))
524 fattr[0] = GNUNET_htonll (sbuf.st_size);
525 fattr[0] = GNUNET_htonll (sbuf.st_mtime);
527 GNUNET_CRYPTO_hash (fattr, sizeof(fattr), &fx[1]);
531 memset (&fx[1], 1, sizeof(struct GNUNET_HashCode));
532 GNUNET_DISK_directory_scan (filename, &determine_id, &fx[1]);
534 /* use hash here to make hierarchical structure distinct from
535 all files on the same level */
536 GNUNET_CRYPTO_hash (fx, sizeof(fx), &ft);
537 /* use XOR here so that order of the files in the directory
539 GNUNET_CRYPTO_hash_xor (&ft, id, id);
545 * Function called with a filename (or directory name) to publish
546 * (if it has changed since the last time we published it). This function
547 * is called for the top-level files only.
549 * @param cls closure, NULL
550 * @param filename complete filename (absolute path)
551 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR during shutdown
554 add_file (void *cls, const char *filename)
557 struct GNUNET_HashCode key;
558 struct GNUNET_HashCode id;
560 if (GNUNET_YES == do_shutdown)
561 return GNUNET_SYSERR;
562 if ((NULL != strstr (filename, "/.auto-share")) ||
563 (NULL != strstr (filename, "\\.auto-share")))
564 return GNUNET_OK; /* skip internal file */
565 GNUNET_CRYPTO_hash (filename, strlen (filename), &key);
566 wi = GNUNET_CONTAINER_multihashmap_get (work_finished, &key);
567 memset (&id, 0, sizeof(struct GNUNET_HashCode));
568 determine_id (&id, filename);
571 if (0 == memcmp (&id, &wi->id, sizeof(struct GNUNET_HashCode)))
572 return GNUNET_OK; /* skip: we did this one already */
573 /* contents changed, need to re-do the directory... */
576 GNUNET_CONTAINER_multihashmap_remove (work_finished, &key, wi));
580 wi = GNUNET_new (struct WorkItem);
581 wi->filename = GNUNET_strdup (filename);
584 GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi);
585 if (GNUNET_YES == do_shutdown)
586 return GNUNET_SYSERR;
592 * Periodically run task to update our view of the directory to share.
600 start_time = GNUNET_TIME_absolute_get ();
601 (void) GNUNET_DISK_directory_scan (dir_name, &add_file, NULL);
602 schedule_next_task ();
607 * Decide what the next task is (working or scanning) and schedule it.
610 schedule_next_task ()
612 struct GNUNET_TIME_Relative delay;
614 if (GNUNET_YES == do_shutdown)
616 GNUNET_assert (NULL == run_task);
617 if (NULL == work_head)
619 /* delay by at most 4h, at least 1s, and otherwise in between depending
620 on how long it took to scan */
621 delay = GNUNET_TIME_absolute_get_duration (start_time);
622 delay = GNUNET_TIME_relative_saturating_multiply (delay, 100);
623 delay = GNUNET_TIME_relative_min (delay, MAX_DELAY);
624 delay = GNUNET_TIME_relative_max (delay, MIN_DELAY);
625 run_task = GNUNET_SCHEDULER_add_delayed (delay, &scan, NULL);
629 run_task = GNUNET_SCHEDULER_add_now (&work, NULL);
635 * Main function that will be run by the scheduler.
638 * @param args remaining command-line arguments
639 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
640 * @param c configuration
646 const struct GNUNET_CONFIGURATION_Handle *c)
648 /* check arguments */
649 if ((NULL == args[0]) || (NULL != args[1]) ||
650 (GNUNET_YES != GNUNET_DISK_directory_test (args[0], GNUNET_YES)))
653 "You must specify one and only one directory name for automatic publication.\n"));
657 cfg_filename = GNUNET_strdup (cfgfile);
660 work_finished = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
662 run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
665 GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL);
670 * Free memory associated with the work item from the work_finished map.
672 * @param cls NULL (unused)
673 * @param key key of the item in the map (unused)
674 * @param value the `struct WorkItem` to free
675 * @return #GNUNET_OK to continue to iterate
678 free_item (void *cls, const struct GNUNET_HashCode *key, void *value)
680 struct WorkItem *wi = value;
682 GNUNET_free (wi->filename);
689 * The main function to automatically publish content to GNUnet.
691 * @param argc number of arguments from the command line
692 * @param argv command line arguments
693 * @return 0 ok, 1 on error
696 main (int argc, char *const *argv)
698 struct GNUNET_GETOPT_CommandLineOption options[] = {
699 GNUNET_GETOPT_option_uint ('a',
703 "set the desired LEVEL of sender-anonymity"),
706 GNUNET_GETOPT_option_flag (
708 "disable-creation-time",
710 "disable adding the creation time to the metadata of the uploaded file"),
711 &do_disable_creation_time),
713 GNUNET_GETOPT_option_flag (
716 gettext_noop ("do not use libextractor to add keywords or metadata"),
719 GNUNET_GETOPT_option_uint ('p',
723 "specify the priority of the content"),
726 GNUNET_GETOPT_option_uint ('r',
730 "set the desired replication LEVEL"),
733 GNUNET_GETOPT_option_verbose (&verbose),
735 GNUNET_GETOPT_OPTION_END
739 struct GNUNET_SIGNAL_Context *shc_chld;
741 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
743 sigpipe = GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
744 GNUNET_assert (NULL != sigpipe);
746 GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
752 "gnunet-auto-share [OPTIONS] FILENAME",
753 gettext_noop ("Automatically publish files from a directory on GNUnet"),
759 if (NULL != work_finished)
761 (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
764 GNUNET_CONTAINER_multihashmap_destroy (work_finished);
766 while (NULL != (wi = work_head))
768 GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
769 GNUNET_free (wi->filename);
772 GNUNET_SIGNAL_handler_uninstall (shc_chld);
774 GNUNET_DISK_pipe_close (sigpipe);
776 GNUNET_free_non_null (cfg_filename);
778 GNUNET_free ((void *) argv);
783 /* end of gnunet-auto-share.c */