2 This file is part of GNUnet.
3 Copyright (C) 2001--2012 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file fs/gnunet-auto-share.c
22 * @brief automatically publish files on GNUnet
23 * @author Christian Grothoff
26 * - support loading meta data / keywords from resource file
27 * - add stability timer (a la buildbot)
30 #include "gnunet_util_lib.h"
32 #define MAX_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_HOURS, 4)
34 #define MIN_DELAY GNUNET_TIME_UNIT_MINUTES
38 * Item in our work queue (or in the set of files/directories
39 * we have successfully published).
43 * PENDING Work is kept in a linked list.
45 struct WorkItem *prev;
48 * PENDING Work is kept in a linked list.
50 struct WorkItem *next;
53 * Filename of the work item.
58 * Unique identity for this work item (used to detect
59 * if we need to do the work again).
61 struct GNUNET_HashCode id;
66 * Global return value from 'main'.
71 * Are we running 'verbosely'?
73 static unsigned int verbose;
76 * Configuration to use.
78 static const struct GNUNET_CONFIGURATION_Handle *cfg;
81 * Name of the configuration file.
83 static char *cfg_filename;
86 * Disable extractor option to use for publishing.
88 static int disable_extractor;
91 * Disable creation time option to use for publishing.
93 static int do_disable_creation_time;
96 * Handle for the main task that does scanning and working.
98 static struct GNUNET_SCHEDULER_Task *run_task;
101 * Anonymity level option to use for publishing.
103 static unsigned int anonymity_level = 1;
106 * Content priority option to use for publishing.
108 static unsigned int content_priority = 365;
111 * Replication level option to use for publishing.
113 static unsigned int replication_level = 1;
116 * Top-level directory we monitor to auto-publish.
118 static const char *dir_name;
121 * Head of linked list of files still to publish.
123 static struct WorkItem *work_head;
126 * Tail of linked list of files still to publish.
128 static struct WorkItem *work_tail;
131 * Map from the hash of the filename (!) to a `struct WorkItem`
134 static struct GNUNET_CONTAINER_MultiHashMap *work_finished;
137 * Set to #GNUNET_YES if we are shutting down.
139 static int do_shutdown;
142 * Start time of the current round; used to determine how long
143 * one iteration takes (which influences how fast we schedule
146 static struct GNUNET_TIME_Absolute start_time;
149 * Pipe used to communicate 'gnunet-publish' completion (SIGCHLD) via signal.
151 static struct GNUNET_DISK_PipeHandle *sigpipe;
154 * Handle to the 'gnunet-publish' process that we executed.
156 static struct GNUNET_OS_Process *publish_proc;
160 * Compute the name of the state database file we will use.
167 GNUNET_asprintf(&ret,
170 (DIR_SEPARATOR == dir_name[strlen(dir_name) - 1])
172 : DIR_SEPARATOR_STR);
178 * Load the set of #work_finished items from disk.
184 struct GNUNET_BIO_ReadHandle *rh;
186 struct GNUNET_HashCode id;
191 fn = get_state_file();
192 rh = GNUNET_BIO_read_open(fn);
197 if (GNUNET_OK != GNUNET_BIO_read_int32(rh, &n))
201 if ((GNUNET_OK != GNUNET_BIO_read_string(rh, "filename", &fn, 1024)) ||
203 GNUNET_BIO_read(rh, "id", &id, sizeof(struct GNUNET_HashCode))))
205 wi = GNUNET_new(struct WorkItem);
208 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
209 "Loaded serialization ID for `%s' is `%s'\n",
213 GNUNET_CRYPTO_hash(wi->filename, strlen(wi->filename), &id);
214 GNUNET_break(GNUNET_OK ==
215 GNUNET_CONTAINER_multihashmap_put(
219 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
221 if (GNUNET_OK == GNUNET_BIO_read_close(rh, &emsg))
225 GNUNET_free_non_null(fn);
227 (void)GNUNET_BIO_read_close(rh, &emsg);
228 GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
229 _("Failed to load state: %s\n"),
231 GNUNET_free_non_null(emsg);
236 * Write work item from the #work_finished map to the given write handle.
238 * @param cls the `struct GNUNET_BIO_WriteHandle *`
239 * @param key key of the item in the map (unused)
240 * @param value the `struct WorkItem` to write
241 * @return #GNUNET_OK to continue to iterate (if write worked)
244 write_item(void *cls, const struct GNUNET_HashCode *key, void *value)
246 struct GNUNET_BIO_WriteHandle *wh = cls;
247 struct WorkItem *wi = value;
249 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
250 "Saving serialization ID of file `%s' with value `%s'\n",
252 GNUNET_h2s(&wi->id));
253 if ((GNUNET_OK != GNUNET_BIO_write_string(wh, wi->filename)) ||
255 GNUNET_BIO_write(wh, &wi->id, sizeof(struct GNUNET_HashCode))))
256 return GNUNET_SYSERR; /* write error, abort iteration */
262 * Save the set of #work_finished items on disk.
268 struct GNUNET_BIO_WriteHandle *wh;
271 n = GNUNET_CONTAINER_multihashmap_size(work_finished);
272 fn = get_state_file();
273 wh = GNUNET_BIO_write_open(fn);
276 GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
277 _("Failed to save state to file %s\n"),
282 if (GNUNET_OK != GNUNET_BIO_write_int32(wh, n))
284 (void)GNUNET_BIO_write_close(wh);
285 GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
286 _("Failed to save state to file %s\n"),
291 (void)GNUNET_CONTAINER_multihashmap_iterate(work_finished, &write_item, wh);
292 if (GNUNET_OK != GNUNET_BIO_write_close(wh))
293 GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
294 _("Failed to save state to file %s\n"),
301 * Task run on shutdown. Serializes our current state to disk.
303 * @param cls closure, unused
306 do_stop_task(void *cls)
308 do_shutdown = GNUNET_YES;
309 if (NULL != publish_proc)
311 GNUNET_OS_process_kill(publish_proc, SIGKILL);
314 if (NULL != run_task)
316 GNUNET_SCHEDULER_cancel(run_task);
323 * Decide what the next task is (working or scanning) and schedule it.
326 schedule_next_task(void);
330 * Task triggered whenever we receive a SIGCHLD (child
333 * @param cls the `struct WorkItem` we were working on
336 maint_child_death(void *cls)
338 struct WorkItem *wi = cls;
339 struct GNUNET_HashCode key;
340 enum GNUNET_OS_ProcessStatusType type;
344 const struct GNUNET_DISK_FileHandle *pr;
345 const struct GNUNET_SCHEDULER_TaskContext *tc;
348 pr = GNUNET_DISK_pipe_handle(sigpipe, GNUNET_DISK_PIPE_END_READ);
349 tc = GNUNET_SCHEDULER_get_task_context();
350 if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
352 /* shutdown scheduled us, someone else will kill child,
353 we should just try again */
354 run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
360 /* consume the signal */
361 GNUNET_break(0 < GNUNET_DISK_file_read(pr, &c, sizeof(c)));
363 ret = GNUNET_OS_process_status(publish_proc, &type, &code);
364 GNUNET_assert(GNUNET_SYSERR != ret);
365 if (GNUNET_NO == ret)
367 /* process still running? Then where did the SIGCHLD come from?
368 Well, let's declare it spurious (kernel bug?) and keep rolling.
371 run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
377 GNUNET_assert(GNUNET_OK == ret);
379 GNUNET_OS_process_destroy(publish_proc);
382 if (GNUNET_YES == do_shutdown)
384 GNUNET_free(wi->filename);
388 if ((GNUNET_OS_PROCESS_EXITED == type) && (0 == code))
390 GNUNET_log(GNUNET_ERROR_TYPE_INFO,
391 _("Publication of `%s' done\n"),
393 GNUNET_CRYPTO_hash(wi->filename, strlen(wi->filename), &key);
394 GNUNET_break(GNUNET_OK ==
395 GNUNET_CONTAINER_multihashmap_put(
399 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
403 GNUNET_CONTAINER_DLL_insert_tail(work_head, work_tail, wi);
406 schedule_next_task();
411 * Signal handler called for SIGCHLD. Triggers the
412 * respective handler by writing to the trigger pipe.
415 sighandler_child_death()
418 int old_errno = errno; /* back-up errno */
422 GNUNET_DISK_file_write(GNUNET_DISK_pipe_handle(sigpipe,
423 GNUNET_DISK_PIPE_END_WRITE),
426 errno = old_errno; /* restore errno */
431 * Function called to process work items.
433 * @param cls closure, NULL
438 static char *argv[14];
439 static char anon_level[20];
440 static char content_prio[20];
441 static char repl_level[20];
443 const struct GNUNET_DISK_FileHandle *pr;
448 GNUNET_CONTAINER_DLL_remove(work_head, work_tail, wi);
450 argv[argc++] = "gnunet-publish";
453 if (disable_extractor)
455 if (do_disable_creation_time)
458 argv[argc++] = cfg_filename;
459 GNUNET_snprintf(anon_level, sizeof(anon_level), "%u", anonymity_level);
461 argv[argc++] = anon_level;
462 GNUNET_snprintf(content_prio, sizeof(content_prio), "%u", content_priority);
464 argv[argc++] = content_prio;
465 GNUNET_snprintf(repl_level, sizeof(repl_level), "%u", replication_level);
467 argv[argc++] = repl_level;
468 argv[argc++] = wi->filename;
470 GNUNET_log(GNUNET_ERROR_TYPE_INFO, _("Publishing `%s'\n"), wi->filename);
471 GNUNET_assert(NULL == publish_proc);
472 publish_proc = GNUNET_OS_start_process_vap(GNUNET_YES,
479 if (NULL == publish_proc)
481 GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
482 _("Failed to run `%s'\n"),
484 GNUNET_CONTAINER_DLL_insert(work_head, work_tail, wi);
486 GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_UNIT_MINUTES, &work, NULL);
489 pr = GNUNET_DISK_pipe_handle(sigpipe, GNUNET_DISK_PIPE_END_READ);
490 run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
498 * Recursively scan the given file/directory structure to determine
499 * a unique ID that represents the current state of the hierarchy.
501 * @param cls where to store the unique ID we are computing
502 * @param filename file to scan
503 * @return #GNUNET_OK (always)
506 determine_id(void *cls, const char *filename)
508 struct GNUNET_HashCode *id = cls;
510 struct GNUNET_HashCode fx[2];
511 struct GNUNET_HashCode ft;
513 if (0 != stat(filename, &sbuf))
515 GNUNET_log_strerror_file(GNUNET_ERROR_TYPE_WARNING, "stat", filename);
518 GNUNET_CRYPTO_hash(filename, strlen(filename), &fx[0]);
519 if (!S_ISDIR(sbuf.st_mode))
523 fattr[0] = GNUNET_htonll(sbuf.st_size);
524 fattr[0] = GNUNET_htonll(sbuf.st_mtime);
526 GNUNET_CRYPTO_hash(fattr, sizeof(fattr), &fx[1]);
530 memset(&fx[1], 1, sizeof(struct GNUNET_HashCode));
531 GNUNET_DISK_directory_scan(filename, &determine_id, &fx[1]);
533 /* use hash here to make hierarchical structure distinct from
534 all files on the same level */
535 GNUNET_CRYPTO_hash(fx, sizeof(fx), &ft);
536 /* use XOR here so that order of the files in the directory
538 GNUNET_CRYPTO_hash_xor(&ft, id, id);
544 * Function called with a filename (or directory name) to publish
545 * (if it has changed since the last time we published it). This function
546 * is called for the top-level files only.
548 * @param cls closure, NULL
549 * @param filename complete filename (absolute path)
550 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR during shutdown
553 add_file(void *cls, const char *filename)
556 struct GNUNET_HashCode key;
557 struct GNUNET_HashCode id;
559 if (GNUNET_YES == do_shutdown)
560 return GNUNET_SYSERR;
561 if ((NULL != strstr(filename, "/.auto-share")) ||
562 (NULL != strstr(filename, "\\.auto-share")))
563 return GNUNET_OK; /* skip internal file */
564 GNUNET_CRYPTO_hash(filename, strlen(filename), &key);
565 wi = GNUNET_CONTAINER_multihashmap_get(work_finished, &key);
566 memset(&id, 0, sizeof(struct GNUNET_HashCode));
567 determine_id(&id, filename);
570 if (0 == memcmp(&id, &wi->id, sizeof(struct GNUNET_HashCode)))
571 return GNUNET_OK; /* skip: we did this one already */
572 /* contents changed, need to re-do the directory... */
575 GNUNET_CONTAINER_multihashmap_remove(work_finished, &key, wi));
579 wi = GNUNET_new(struct WorkItem);
580 wi->filename = GNUNET_strdup(filename);
583 GNUNET_CONTAINER_DLL_insert(work_head, work_tail, wi);
584 if (GNUNET_YES == do_shutdown)
585 return GNUNET_SYSERR;
591 * Periodically run task to update our view of the directory to share.
599 start_time = GNUNET_TIME_absolute_get();
600 (void)GNUNET_DISK_directory_scan(dir_name, &add_file, NULL);
601 schedule_next_task();
606 * Decide what the next task is (working or scanning) and schedule it.
611 struct GNUNET_TIME_Relative delay;
613 if (GNUNET_YES == do_shutdown)
615 GNUNET_assert(NULL == run_task);
616 if (NULL == work_head)
618 /* delay by at most 4h, at least 1s, and otherwise in between depending
619 on how long it took to scan */
620 delay = GNUNET_TIME_absolute_get_duration(start_time);
621 delay = GNUNET_TIME_relative_saturating_multiply(delay, 100);
622 delay = GNUNET_TIME_relative_min(delay, MAX_DELAY);
623 delay = GNUNET_TIME_relative_max(delay, MIN_DELAY);
624 run_task = GNUNET_SCHEDULER_add_delayed(delay, &scan, NULL);
628 run_task = GNUNET_SCHEDULER_add_now(&work, NULL);
634 * Main function that will be run by the scheduler.
637 * @param args remaining command-line arguments
638 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
639 * @param c configuration
645 const struct GNUNET_CONFIGURATION_Handle *c)
647 /* check arguments */
648 if ((NULL == args[0]) || (NULL != args[1]) ||
649 (GNUNET_YES != GNUNET_DISK_directory_test(args[0], GNUNET_YES)))
652 "You must specify one and only one directory name for automatic publication.\n"));
656 cfg_filename = GNUNET_strdup(cfgfile);
659 work_finished = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO);
661 run_task = GNUNET_SCHEDULER_add_with_priority(GNUNET_SCHEDULER_PRIORITY_IDLE,
664 GNUNET_SCHEDULER_add_shutdown(&do_stop_task, NULL);
669 * Free memory associated with the work item from the work_finished map.
671 * @param cls NULL (unused)
672 * @param key key of the item in the map (unused)
673 * @param value the `struct WorkItem` to free
674 * @return #GNUNET_OK to continue to iterate
677 free_item(void *cls, const struct GNUNET_HashCode *key, void *value)
679 struct WorkItem *wi = value;
681 GNUNET_free(wi->filename);
688 * The main function to automatically publish content to GNUnet.
690 * @param argc number of arguments from the command line
691 * @param argv command line arguments
692 * @return 0 ok, 1 on error
695 main(int argc, char *const *argv)
697 struct GNUNET_GETOPT_CommandLineOption options[] = {
698 GNUNET_GETOPT_option_uint('a',
702 "set the desired LEVEL of sender-anonymity"),
705 GNUNET_GETOPT_option_flag(
707 "disable-creation-time",
709 "disable adding the creation time to the metadata of the uploaded file"),
710 &do_disable_creation_time),
712 GNUNET_GETOPT_option_flag(
715 gettext_noop("do not use libextractor to add keywords or metadata"),
718 GNUNET_GETOPT_option_uint('p',
722 "specify the priority of the content"),
725 GNUNET_GETOPT_option_uint('r',
729 "set the desired replication LEVEL"),
732 GNUNET_GETOPT_option_verbose(&verbose),
734 GNUNET_GETOPT_OPTION_END
738 struct GNUNET_SIGNAL_Context *shc_chld;
740 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args(argc, argv, &argc, &argv))
742 sigpipe = GNUNET_DISK_pipe(GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
743 GNUNET_assert(NULL != sigpipe);
745 GNUNET_SIGNAL_handler_install(GNUNET_SIGCHLD, &sighandler_child_death);
751 "gnunet-auto-share [OPTIONS] FILENAME",
752 gettext_noop("Automatically publish files from a directory on GNUnet"),
758 if (NULL != work_finished)
760 (void)GNUNET_CONTAINER_multihashmap_iterate(work_finished,
763 GNUNET_CONTAINER_multihashmap_destroy(work_finished);
765 while (NULL != (wi = work_head))
767 GNUNET_CONTAINER_DLL_remove(work_head, work_tail, wi);
768 GNUNET_free(wi->filename);
771 GNUNET_SIGNAL_handler_uninstall(shc_chld);
773 GNUNET_DISK_pipe_close(sigpipe);
775 GNUNET_free_non_null(cfg_filename);
777 GNUNET_free((void *)argv);
781 /* end of gnunet-auto-share.c */