2 This file is part of GNUnet.
3 (C) 2001-2013 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/gnunet-publish.c
22 * @brief publishing files on GNUnet
23 * @author Christian Grothoff
24 * @author Krista Bennett
25 * @author James Blackwell
26 * @author Igor Wronsky
29 #include "gnunet_fs_service.h"
32 * Global return value from 'main'.
37 * Command line option 'verbose' set
42 * Handle to our configuration.
44 static const struct GNUNET_CONFIGURATION_Handle *cfg;
47 * Handle for interaction with file-sharing service.
49 static struct GNUNET_FS_Handle *ctx;
52 * Handle to FS-publishing operation.
54 static struct GNUNET_FS_PublishContext *pc;
57 * Meta-data provided via command-line option.
59 static struct GNUNET_CONTAINER_MetaData *meta;
62 * Keywords provided via command-line option.
64 static struct GNUNET_FS_Uri *topKeywords;
67 * Options we set for published blocks.
69 static struct GNUNET_FS_BlockOptions bo = { {0LL}, 1, 365, 1 };
72 * Value of URI provided on command-line (when not publishing
73 * a file but just creating UBlocks to refer to an existing URI).
75 static char *uri_string;
78 * Value of URI provided on command-line (when not publishing
79 * a file but just creating UBlocks to refer to an existing URI);
80 * parsed version of 'uri_string'.
82 static struct GNUNET_FS_Uri *uri;
85 * Command-line option for namespace publishing: identifier for updates
86 * to this publication.
91 * Command-line option for namespace publishing: identifier for this
97 * Command-line option identifying the pseudonym to use for the publication.
99 static char *pseudonym;
102 * Command-line option for 'inserting'
104 static int do_insert;
107 * Command-line option to disable meta data extraction.
109 static int disable_extractor;
112 * Command-line option to merely simulate publishing operation.
114 static int do_simulate;
117 * Command-line option to only perform meta data extraction, but not publish.
119 static int extract_only;
122 * Command-line option to disable adding creation time.
124 static int do_disable_creation_time;
127 * Task run on CTRL-C to kill everything nicely.
129 static GNUNET_SCHEDULER_TaskIdentifier kill_task;
132 * Handle to the directory scanner (for recursive insertions).
134 static struct GNUNET_FS_DirScanner *ds;
137 * Which namespace do we publish to? NULL if we do not publish to
140 static struct GNUNET_FS_Namespace *namespace;
144 * We are finished with the publishing operation, clean up all
148 * @param tc scheduler context
151 do_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
153 struct GNUNET_FS_PublishContext *p;
155 kill_task = GNUNET_SCHEDULER_NO_TASK;
160 GNUNET_FS_publish_stop (p);
164 GNUNET_CONTAINER_meta_data_destroy (meta);
171 * Stop the directory scanner (we had an error).
174 * @param tc scheduler context
177 stop_scanner_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
179 kill_task = GNUNET_SCHEDULER_NO_TASK;
182 GNUNET_FS_directory_scan_abort (ds);
185 if (NULL != namespace)
187 GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
190 GNUNET_FS_stop (ctx);
197 * Called by FS client to give information about the progress of an
201 * @param info details about the event, specifying the event type
202 * and various bits about the event
203 * @return client-context (for the next progress call
204 * for this operation; should be set to NULL for
205 * SUSPEND and STOPPED events). The value returned
206 * will be passed to future callbacks in the respective
207 * field in the GNUNET_FS_ProgressInfo struct.
210 progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info)
215 switch (info->status)
217 case GNUNET_FS_STATUS_PUBLISH_START:
219 case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
222 s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.eta,
224 FPRINTF (stdout, _("Publishing `%s' at %llu/%llu (%s remaining)\n"),
225 info->value.publish.filename,
226 (unsigned long long) info->value.publish.completed,
227 (unsigned long long) info->value.publish.size, s);
230 case GNUNET_FS_STATUS_PUBLISH_ERROR:
231 FPRINTF (stderr, _("Error publishing: %s.\n"),
232 info->value.publish.specifics.error.message);
233 if (kill_task != GNUNET_SCHEDULER_NO_TASK)
235 GNUNET_SCHEDULER_cancel (kill_task);
236 kill_task = GNUNET_SCHEDULER_NO_TASK;
238 kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL);
240 case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
241 FPRINTF (stdout, _("Publishing `%s' done.\n"),
242 info->value.publish.filename);
243 suri = GNUNET_FS_uri_to_string (info->value.publish.specifics.
245 FPRINTF (stdout, _("URI is `%s'.\n"), suri);
247 if (NULL == info->value.publish.pctx)
249 if (GNUNET_SCHEDULER_NO_TASK != kill_task)
251 GNUNET_SCHEDULER_cancel (kill_task);
252 kill_task = GNUNET_SCHEDULER_NO_TASK;
254 kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL);
258 case GNUNET_FS_STATUS_PUBLISH_STOPPED:
259 GNUNET_break (NULL == pc);
261 case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
263 case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
264 FPRINTF (stderr, "%s", _("Cleanup after abort complete.\n"));
267 FPRINTF (stderr, _("Unexpected status: %d\n"), info->status);
270 return ""; /* non-null */
275 * Print metadata entries (except binary
276 * metadata and the filename).
279 * @param plugin_name name of the plugin that generated the meta data
280 * @param type type of the meta data
281 * @param format format of data
282 * @param data_mime_type mime type of data
283 * @param data value of the meta data
284 * @param data_size number of bytes in data
288 meta_printer (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type,
289 enum EXTRACTOR_MetaFormat format, const char *data_mime_type,
290 const char *data, size_t data_size)
292 if ((EXTRACTOR_METAFORMAT_UTF8 != format) &&
293 (EXTRACTOR_METAFORMAT_C_STRING != format))
295 if (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type)
297 FPRINTF (stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string (type), data);
303 * Iterator printing keywords
306 * @param keyword the keyword
307 * @param is_mandatory is the keyword mandatory (in a search)
308 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
311 keyword_printer (void *cls, const char *keyword, int is_mandatory)
313 FPRINTF (stdout, "\t%s\n", keyword);
319 * Function called on all entries before the publication. This is
320 * where we perform modifications to the default based on command-line
324 * @param fi the entry in the publish-structure
325 * @param length length of the file or directory
326 * @param m metadata for the file or directory (can be modified)
327 * @param uri pointer to the keywords that will be used for this entry (can be modified)
328 * @param bo block options
329 * @param do_index should we index?
330 * @param client_info pointer to client context set upon creation (can be modified)
331 * @return GNUNET_OK to continue, GNUNET_NO to remove
332 * this entry from the directory, GNUNET_SYSERR
333 * to abort the iteration
336 publish_inspector (void *cls, struct GNUNET_FS_FileInformation *fi,
337 uint64_t length, struct GNUNET_CONTAINER_MetaData *m,
338 struct GNUNET_FS_Uri **uri,
339 struct GNUNET_FS_BlockOptions *bo, int *do_index,
344 struct GNUNET_FS_Uri *new_uri;
348 if ( (disable_extractor) &&
351 GNUNET_FS_uri_destroy (*uri);
354 if (NULL != topKeywords)
358 new_uri = GNUNET_FS_uri_ksk_merge (topKeywords, *uri);
359 GNUNET_FS_uri_destroy (*uri);
361 GNUNET_FS_uri_destroy (topKeywords);
371 GNUNET_CONTAINER_meta_data_merge (m, meta);
372 GNUNET_CONTAINER_meta_data_destroy (meta);
375 if (!do_disable_creation_time)
376 GNUNET_CONTAINER_meta_data_add_publication_date (m);
379 fn = GNUNET_CONTAINER_meta_data_get_by_type (m,
380 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
381 fs = GNUNET_STRINGS_byte_size_fancy (length);
382 FPRINTF (stdout, _("Meta data for file `%s' (%s)\n"), fn, fs);
383 GNUNET_CONTAINER_meta_data_iterate (m, &meta_printer, NULL);
384 FPRINTF (stdout, _("Keywords for file `%s' (%s)\n"), fn, fs);
388 GNUNET_FS_uri_ksk_get_keywords (*uri, &keyword_printer, NULL);
389 FPRINTF (stdout, "%s", "\n");
391 if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (m))
392 GNUNET_FS_file_information_inspect (fi, &publish_inspector, fi);
398 * Function called upon completion of the publishing
399 * of the UBLOCK for the SKS URI. As this is the last
400 * step, stop our interaction with FS (clean up).
402 * @param cls NULL (closure)
403 * @param sks_uri URI for the block that was published
404 * @param emsg error message, NULL on success
407 uri_sks_continuation (void *cls, const struct GNUNET_FS_Uri *sks_uri,
412 FPRINTF (stderr, "%s\n", emsg);
415 GNUNET_FS_uri_destroy (uri);
417 GNUNET_FS_stop (ctx);
423 * Function called upon completion of the publishing
424 * of the UBLOCK for the KSK URI. Continue with
425 * publishing the SKS URI (if applicable) or clean up.
427 * @param cls NULL (closure)
428 * @param ksk_uri URI for the block that was published
429 * @param emsg error message, NULL on success
432 uri_ksk_continuation (void *cls, const struct GNUNET_FS_Uri *ksk_uri,
435 struct GNUNET_FS_Namespace *ns;
439 FPRINTF (stderr, "%s\n", emsg);
442 if (NULL != pseudonym)
444 ns = GNUNET_FS_namespace_create (ctx, pseudonym);
447 FPRINTF (stderr, _("Failed to create namespace `%s' (illegal filename?)\n"), pseudonym);
452 GNUNET_FS_publish_sks (ctx, ns, this_id, next_id, meta, uri, &bo,
453 GNUNET_FS_PUBLISH_OPTION_NONE,
454 &uri_sks_continuation, NULL);
455 GNUNET_assert (GNUNET_OK == GNUNET_FS_namespace_delete (ns, GNUNET_NO));
459 GNUNET_FS_uri_destroy (uri);
461 GNUNET_FS_stop (ctx);
467 * Iterate over the results from the directory scan and extract
468 * the desired information for the publishing operation.
470 * @param item root with the data from the directroy scan
471 * @return handle with the information for the publishing operation
473 static struct GNUNET_FS_FileInformation *
474 get_file_information (struct GNUNET_FS_ShareTreeItem *item)
476 struct GNUNET_FS_FileInformation *fi;
477 struct GNUNET_FS_FileInformation *fic;
478 struct GNUNET_FS_ShareTreeItem *child;
480 if (GNUNET_YES == item->is_directory)
482 GNUNET_CONTAINER_meta_data_delete (item->meta,
483 EXTRACTOR_METATYPE_MIMETYPE,
485 GNUNET_FS_meta_data_make_directory (item->meta);
486 if (NULL == item->ksk_uri)
488 const char *mime = GNUNET_FS_DIRECTORY_MIME;
489 item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &mime);
492 GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri, GNUNET_FS_DIRECTORY_MIME,
494 fi = GNUNET_FS_file_information_create_empty_directory (ctx, NULL,
497 &bo, item->filename);
498 for (child = item->children_head; child; child = child->next)
500 fic = get_file_information (child);
501 GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic));
506 fi = GNUNET_FS_file_information_create_from_file (ctx, NULL,
508 item->ksk_uri, item->meta,
517 * We've finished scanning the directory and optimized the meta data.
518 * Begin the publication process.
520 * @param directroy_scan_result result from the directory scan, freed in this function
523 directory_trim_complete (struct GNUNET_FS_ShareTreeItem *directory_scan_result)
525 struct GNUNET_FS_FileInformation *fi;
527 fi = get_file_information (directory_scan_result);
528 GNUNET_FS_share_tree_free (directory_scan_result);
531 FPRINTF (stderr, "%s", _("Could not publish\n"));
532 if (NULL != namespace)
533 GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
534 GNUNET_FS_stop (ctx);
538 GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL);
541 if (NULL != namespace)
542 GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
543 GNUNET_FS_file_information_destroy (fi, NULL, NULL);
544 GNUNET_FS_stop (ctx);
545 if (GNUNET_SCHEDULER_NO_TASK != kill_task)
547 GNUNET_SCHEDULER_cancel (kill_task);
548 kill_task = GNUNET_SCHEDULER_NO_TASK;
552 pc = GNUNET_FS_publish_start (ctx, fi, namespace, this_id, next_id,
554 GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY :
555 GNUNET_FS_PUBLISH_OPTION_NONE);
558 FPRINTF (stderr, "%s", _("Could not start publishing.\n"));
559 GNUNET_FS_stop (ctx);
567 * Function called by the directory scanner as we build the tree
568 * that we will need to publish later.
571 * @param filename which file we are making progress on
572 * @param is_directory GNUNET_YES if this is a directory,
573 * GNUNET_NO if this is a file
574 * GNUNET_SYSERR if it is neither (or unknown)
575 * @param reason kind of progress we are making
578 directory_scan_cb (void *cls,
579 const char *filename,
581 enum GNUNET_FS_DirScannerProgressUpdateReason reason)
583 struct GNUNET_FS_ShareTreeItem *directory_scan_result;
587 case GNUNET_FS_DIRSCANNER_FILE_START:
590 if (is_directory == GNUNET_YES)
591 FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename);
593 FPRINTF (stdout, _("Scanning file `%s'.\n"), filename);
596 case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
598 _("There was trouble processing file `%s', skipping it.\n"),
601 case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
603 FPRINTF (stdout, "%s", _("Preprocessing complete.\n"));
605 case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
607 FPRINTF (stdout, _("Extracting meta data from file `%s' complete.\n"), filename);
609 case GNUNET_FS_DIRSCANNER_FINISHED:
611 FPRINTF (stdout, "%s", _("Meta data extraction has finished.\n"));
612 directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
614 GNUNET_FS_share_tree_trim (directory_scan_result);
615 directory_trim_complete (directory_scan_result);
617 case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
618 FPRINTF (stdout, "%s", _("Internal error scanning directory.\n"));
619 if (kill_task != GNUNET_SCHEDULER_NO_TASK)
621 GNUNET_SCHEDULER_cancel (kill_task);
622 kill_task = GNUNET_SCHEDULER_NO_TASK;
624 kill_task = GNUNET_SCHEDULER_add_now (&stop_scanner_task, NULL);
635 * Main function that will be run by the scheduler.
638 * @param args remaining command-line arguments
639 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
640 * @param c configuration
643 run (void *cls, char *const *args, const char *cfgfile,
644 const struct GNUNET_CONFIGURATION_Handle *c)
649 /* check arguments */
650 if ((NULL != uri_string) && (extract_only))
652 printf (_("Cannot extract metadata from a URI!\n"));
656 if (((NULL == uri_string) || (extract_only)) &&
657 ((NULL == args[0]) || (NULL != args[1])))
659 printf (_("You must specify one and only one filename for insertion.\n"));
663 if ((NULL != uri_string) && (NULL != args[0]))
665 printf (_("You must NOT specify an URI and a filename.\n"));
669 if (NULL != pseudonym)
673 FPRINTF (stderr, _("Option `%s' is required when using option `%s'.\n"),
680 { /* ordinary insertion checks */
683 FPRINTF (stderr, _("Option `%s' makes no sense without option `%s'.\n"),
690 FPRINTF (stderr, _("Option `%s' makes no sense without option `%s'.\n"),
698 GNUNET_FS_start (cfg, "gnunet-publish", &progress_cb, NULL,
699 GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
702 FPRINTF (stderr, _("Could not initialize `%s' subsystem.\n"), "FS");
707 if (NULL != pseudonym)
709 namespace = GNUNET_FS_namespace_create (ctx, pseudonym);
710 if (NULL == namespace)
712 FPRINTF (stderr, _("Failed to create namespace `%s' (illegal filename?)\n"), pseudonym);
713 GNUNET_FS_stop (ctx);
718 if (NULL != uri_string)
721 if (NULL == (uri = GNUNET_FS_uri_parse (uri_string, &emsg)))
723 FPRINTF (stderr, _("Failed to parse URI: %s\n"), emsg);
725 if (namespace != NULL)
726 GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
727 GNUNET_FS_stop (ctx);
731 GNUNET_FS_publish_ksk (ctx, topKeywords, meta, uri, &bo,
732 GNUNET_FS_PUBLISH_OPTION_NONE, &uri_ksk_continuation,
734 if (NULL != namespace)
735 GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
739 GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex))
741 if (0 != ACCESS (args[0], R_OK))
744 _("Failed to access `%s': %s\n"),
749 ds = GNUNET_FS_directory_scan_start (args[0],
752 &directory_scan_cb, NULL);
756 "%s", _("Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n"));
760 GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task,
766 * The main function to publish content to GNUnet.
768 * @param argc number of arguments from the command line
769 * @param argv command line arguments
770 * @return 0 ok, 1 on error
773 main (int argc, char *const *argv)
775 static const struct GNUNET_GETOPT_CommandLineOption options[] = {
776 {'a', "anonymity", "LEVEL",
777 gettext_noop ("set the desired LEVEL of sender-anonymity"),
778 1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level},
779 {'d', "disable-creation-time", NULL,
781 ("disable adding the creation time to the metadata of the uploaded file"),
782 0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
783 {'D', "disable-extractor", NULL,
784 gettext_noop ("do not use libextractor to add keywords or metadata"),
785 0, &GNUNET_GETOPT_set_one, &disable_extractor},
786 {'e', "extract", NULL,
788 ("print list of extracted keywords that would be used, but do not perform upload"),
789 0, &GNUNET_GETOPT_set_one, &extract_only},
790 {'k', "key", "KEYWORD",
792 ("add an additional keyword for the top-level file or directory"
793 " (this option can be specified multiple times)"),
794 1, &GNUNET_FS_getopt_set_keywords, &topKeywords},
795 {'m', "meta", "TYPE:VALUE",
796 gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
797 1, &GNUNET_FS_getopt_set_metadata, &meta},
798 {'n', "noindex", NULL,
799 gettext_noop ("do not index, perform full insertion (stores entire "
800 "file in encrypted form in GNUnet database)"),
801 0, &GNUNET_GETOPT_set_one, &do_insert},
804 ("specify ID of an updated version to be published in the future"
805 " (for namespace insertions only)"),
806 1, &GNUNET_GETOPT_set_string, &next_id},
807 {'p', "priority", "PRIORITY",
808 gettext_noop ("specify the priority of the content"),
809 1, &GNUNET_GETOPT_set_uint, &bo.content_priority},
810 {'P', "pseudonym", "NAME",
812 ("publish the files under the pseudonym NAME (place file into namespace)"),
813 1, &GNUNET_GETOPT_set_string, &pseudonym},
814 {'r', "replication", "LEVEL",
815 gettext_noop ("set the desired replication LEVEL"),
816 1, &GNUNET_GETOPT_set_uint, &bo.replication_level},
817 {'s', "simulate-only", NULL,
818 gettext_noop ("only simulate the process but do not do any "
819 "actual publishing (useful to compute URIs)"),
820 0, &GNUNET_GETOPT_set_one, &do_simulate},
822 gettext_noop ("set the ID of this version of the publication"
823 " (for namespace insertions only)"),
824 1, &GNUNET_GETOPT_set_string, &this_id},
826 gettext_noop ("URI to be published (can be used instead of passing a "
827 "file to add keywords to the file with the respective URI)"),
828 1, &GNUNET_GETOPT_set_string, &uri_string},
829 {'V', "verbose", NULL,
830 gettext_noop ("be verbose (print progress information)"),
831 0, &GNUNET_GETOPT_set_one, &verbose},
832 GNUNET_GETOPT_OPTION_END
835 GNUNET_FS_year_to_time (GNUNET_FS_get_current_year () + 2);
837 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
840 GNUNET_PROGRAM_run (argc, argv, "gnunet-publish [OPTIONS] FILENAME",
842 ("Publish a file or directory on GNUnet"),
843 options, &run, NULL)) ? ret : 1;
844 GNUNET_free ((void*) argv);
848 /* end of gnunet-publish.c */