2 This file is part of GNUnet.
3 Copyright (C) 2001-2013 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/gnunet-publish.c
22 * @brief publishing files on GNUnet
23 * @author Christian Grothoff
24 * @author Krista Bennett
25 * @author James Blackwell
26 * @author Igor Wronsky
29 #include "gnunet_fs_service.h"
30 #include "gnunet_identity_service.h"
33 * Global return value from 'main'.
38 * Command line option 'verbose' set
43 * Handle to our configuration.
45 static const struct GNUNET_CONFIGURATION_Handle *cfg;
48 * Handle for interaction with file-sharing service.
50 static struct GNUNET_FS_Handle *ctx;
53 * Handle to FS-publishing operation.
55 static struct GNUNET_FS_PublishContext *pc;
58 * Meta-data provided via command-line option.
60 static struct GNUNET_CONTAINER_MetaData *meta;
63 * Keywords provided via command-line option.
65 static struct GNUNET_FS_Uri *topKeywords;
68 * Options we set for published blocks.
70 static struct GNUNET_FS_BlockOptions bo = { {0LL}, 1, 365, 1 };
73 * Value of URI provided on command-line (when not publishing
74 * a file but just creating UBlocks to refer to an existing URI).
76 static char *uri_string;
79 * Value of URI provided on command-line (when not publishing
80 * a file but just creating UBlocks to refer to an existing URI);
81 * parsed version of 'uri_string'.
83 static struct GNUNET_FS_Uri *uri;
86 * Command-line option for namespace publishing: identifier for updates
87 * to this publication.
92 * Command-line option for namespace publishing: identifier for this
98 * Command-line option identifying the pseudonym to use for the publication.
100 static char *pseudonym;
103 * Command-line option for 'inserting'
105 static int do_insert;
108 * Command-line option to disable meta data extraction.
110 static int disable_extractor;
113 * Command-line option to merely simulate publishing operation.
115 static int do_simulate;
118 * Command-line option to only perform meta data extraction, but not publish.
120 static int extract_only;
123 * Command-line option to disable adding creation time.
125 static int do_disable_creation_time;
128 * Task run on CTRL-C to kill everything nicely.
130 static struct GNUNET_SCHEDULER_Task * kill_task;
133 * Handle to the directory scanner (for recursive insertions).
135 static struct GNUNET_FS_DirScanner *ds;
138 * Which namespace do we publish to? NULL if we do not publish to
141 static struct GNUNET_IDENTITY_Ego *namespace;
144 * Handle to identity service.
146 static struct GNUNET_IDENTITY_Handle *identity;
150 * We are finished with the publishing operation, clean up all
154 * @param tc scheduler context
157 do_stop_task (void *cls,
158 const struct GNUNET_SCHEDULER_TaskContext *tc)
160 struct GNUNET_FS_PublishContext *p;
163 if (NULL != identity)
165 GNUNET_IDENTITY_disconnect (identity);
172 GNUNET_FS_publish_stop (p);
176 GNUNET_CONTAINER_meta_data_destroy (meta);
183 * Stop the directory scanner (we had an error).
186 * @param tc scheduler context
189 stop_scanner_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
194 GNUNET_FS_directory_scan_abort (ds);
197 if (NULL != identity)
199 GNUNET_IDENTITY_disconnect (identity);
202 GNUNET_FS_stop (ctx);
209 * Called by FS client to give information about the progress of an
213 * @param info details about the event, specifying the event type
214 * and various bits about the event
215 * @return client-context (for the next progress call
216 * for this operation; should be set to NULL for
217 * SUSPEND and STOPPED events). The value returned
218 * will be passed to future callbacks in the respective
219 * field in the GNUNET_FS_ProgressInfo struct.
222 progress_cb (void *cls,
223 const struct GNUNET_FS_ProgressInfo *info)
228 switch (info->status)
230 case GNUNET_FS_STATUS_PUBLISH_START:
232 case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
235 s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.eta,
238 _("Publishing `%s' at %llu/%llu (%s remaining)\n"),
239 info->value.publish.filename,
240 (unsigned long long) info->value.publish.completed,
241 (unsigned long long) info->value.publish.size, s);
244 case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
247 s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.specifics.progress_directory.eta,
250 _("Publishing `%s' at %llu/%llu (%s remaining)\n"),
251 info->value.publish.filename,
252 (unsigned long long) info->value.publish.specifics.progress_directory.completed,
253 (unsigned long long) info->value.publish.specifics.progress_directory.total, s);
256 case GNUNET_FS_STATUS_PUBLISH_ERROR:
257 FPRINTF (stderr, _("Error publishing: %s.\n"),
258 info->value.publish.specifics.error.message);
259 if (kill_task != NULL)
261 GNUNET_SCHEDULER_cancel (kill_task);
264 kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL);
266 case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
268 _("Publishing `%s' done.\n"),
269 info->value.publish.filename);
270 suri = GNUNET_FS_uri_to_string (info->value.publish.specifics.
276 if (NULL != info->value.publish.specifics.completed.sks_uri)
278 suri = GNUNET_FS_uri_to_string (info->value.publish.specifics.
281 _("Namespace URI is `%s'.\n"),
285 if (NULL == info->value.publish.pctx)
287 if (NULL != kill_task)
288 GNUNET_SCHEDULER_cancel (kill_task);
289 kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL);
293 case GNUNET_FS_STATUS_PUBLISH_STOPPED:
294 GNUNET_break (NULL == pc);
296 case GNUNET_FS_STATUS_UNINDEX_START:
299 _("Starting cleanup after abort\n"));
301 case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
303 case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
306 _("Cleanup after abort completed.\n"));
307 GNUNET_FS_unindex_stop (info->value.unindex.uc);
309 case GNUNET_FS_STATUS_UNINDEX_ERROR:
312 _("Cleanup after abort failed.\n"));
313 GNUNET_FS_unindex_stop (info->value.unindex.uc);
315 case GNUNET_FS_STATUS_UNINDEX_STOPPED:
319 _("Unexpected status: %d\n"),
323 return ""; /* non-null */
328 * Print metadata entries (except binary
329 * metadata and the filename).
332 * @param plugin_name name of the plugin that generated the meta data
333 * @param type type of the meta data
334 * @param format format of data
335 * @param data_mime_type mime type of @a data
336 * @param data value of the meta data
337 * @param data_size number of bytes in @a data
341 meta_printer (void *cls,
342 const char *plugin_name,
343 enum EXTRACTOR_MetaType type,
344 enum EXTRACTOR_MetaFormat format,
345 const char *data_mime_type,
346 const char *data, size_t data_size)
348 if ((EXTRACTOR_METAFORMAT_UTF8 != format) &&
349 (EXTRACTOR_METAFORMAT_C_STRING != format))
351 if (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type)
353 #if HAVE_LIBEXTRACTOR
354 FPRINTF (stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string (type), data);
356 FPRINTF (stdout, "\t%d - %s\n", type, data);
363 * Iterator printing keywords
366 * @param keyword the keyword
367 * @param is_mandatory is the keyword mandatory (in a search)
368 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to abort
371 keyword_printer (void *cls,
375 FPRINTF (stdout, "\t%s\n", keyword);
381 * Function called on all entries before the publication. This is
382 * where we perform modifications to the default based on command-line
386 * @param fi the entry in the publish-structure
387 * @param length length of the file or directory
388 * @param m metadata for the file or directory (can be modified)
389 * @param uri pointer to the keywords that will be used for this entry (can be modified)
390 * @param bo block options
391 * @param do_index should we index?
392 * @param client_info pointer to client context set upon creation (can be modified)
393 * @return #GNUNET_OK to continue, #GNUNET_NO to remove
394 * this entry from the directory, #GNUNET_SYSERR
395 * to abort the iteration
398 publish_inspector (void *cls,
399 struct GNUNET_FS_FileInformation *fi,
401 struct GNUNET_CONTAINER_MetaData *m,
402 struct GNUNET_FS_Uri **uri,
403 struct GNUNET_FS_BlockOptions *bo,
409 struct GNUNET_FS_Uri *new_uri;
413 if ( (disable_extractor) &&
416 GNUNET_FS_uri_destroy (*uri);
419 if (NULL != topKeywords)
423 new_uri = GNUNET_FS_uri_ksk_merge (topKeywords, *uri);
424 GNUNET_FS_uri_destroy (*uri);
426 GNUNET_FS_uri_destroy (topKeywords);
436 GNUNET_CONTAINER_meta_data_merge (m, meta);
437 GNUNET_CONTAINER_meta_data_destroy (meta);
440 if (!do_disable_creation_time)
441 GNUNET_CONTAINER_meta_data_add_publication_date (m);
444 fn = GNUNET_CONTAINER_meta_data_get_by_type (m,
445 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
446 fs = GNUNET_STRINGS_byte_size_fancy (length);
447 FPRINTF (stdout, _("Meta data for file `%s' (%s)\n"), fn, fs);
448 GNUNET_CONTAINER_meta_data_iterate (m, &meta_printer, NULL);
449 FPRINTF (stdout, _("Keywords for file `%s' (%s)\n"), fn, fs);
453 GNUNET_FS_uri_ksk_get_keywords (*uri, &keyword_printer, NULL);
454 FPRINTF (stdout, "%s", "\n");
456 if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (m))
457 GNUNET_FS_file_information_inspect (fi, &publish_inspector, fi);
463 * Function called upon completion of the publishing
464 * of the UBLOCK for the SKS URI. As this is the last
465 * step, stop our interaction with FS (clean up).
467 * @param cls NULL (closure)
468 * @param sks_uri URI for the block that was published
469 * @param emsg error message, NULL on success
472 uri_sks_continuation (void *cls,
473 const struct GNUNET_FS_Uri *sks_uri,
478 FPRINTF (stderr, "%s\n", emsg);
481 GNUNET_FS_uri_destroy (uri);
483 GNUNET_FS_stop (ctx);
489 * Function called upon completion of the publishing
490 * of the UBLOCK for the KSK URI. Continue with
491 * publishing the SKS URI (if applicable) or clean up.
493 * @param cls NULL (closure)
494 * @param ksk_uri URI for the block that was published
495 * @param emsg error message, NULL on success
498 uri_ksk_continuation (void *cls,
499 const struct GNUNET_FS_Uri *ksk_uri,
502 const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;
506 FPRINTF (stderr, "%s\n", emsg);
509 if (NULL != namespace)
511 priv = GNUNET_IDENTITY_ego_get_private_key (namespace);
512 GNUNET_FS_publish_sks (ctx, priv, this_id, next_id, meta, uri, &bo,
513 GNUNET_FS_PUBLISH_OPTION_NONE,
514 &uri_sks_continuation, NULL);
517 GNUNET_FS_uri_destroy (uri);
519 GNUNET_FS_stop (ctx);
525 * Iterate over the results from the directory scan and extract
526 * the desired information for the publishing operation.
528 * @param item root with the data from the directroy scan
529 * @return handle with the information for the publishing operation
531 static struct GNUNET_FS_FileInformation *
532 get_file_information (struct GNUNET_FS_ShareTreeItem *item)
534 struct GNUNET_FS_FileInformation *fi;
535 struct GNUNET_FS_FileInformation *fic;
536 struct GNUNET_FS_ShareTreeItem *child;
538 if (GNUNET_YES == item->is_directory)
540 if (NULL == item->meta)
541 item->meta = GNUNET_CONTAINER_meta_data_create ();
542 GNUNET_CONTAINER_meta_data_delete (item->meta,
543 EXTRACTOR_METATYPE_MIMETYPE,
545 GNUNET_FS_meta_data_make_directory (item->meta);
546 if (NULL == item->ksk_uri)
548 const char *mime = GNUNET_FS_DIRECTORY_MIME;
549 item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &mime);
552 GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri, GNUNET_FS_DIRECTORY_MIME,
554 fi = GNUNET_FS_file_information_create_empty_directory (ctx, NULL,
557 &bo, item->filename);
558 for (child = item->children_head; child; child = child->next)
560 fic = get_file_information (child);
561 GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic));
566 fi = GNUNET_FS_file_information_create_from_file (ctx, NULL,
568 item->ksk_uri, item->meta,
577 * We've finished scanning the directory and optimized the meta data.
578 * Begin the publication process.
580 * @param directory_scan_result result from the directory scan, freed in this function
583 directory_trim_complete (struct GNUNET_FS_ShareTreeItem *directory_scan_result)
585 struct GNUNET_FS_FileInformation *fi;
586 const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;
588 fi = get_file_information (directory_scan_result);
589 GNUNET_FS_share_tree_free (directory_scan_result);
594 _("Could not publish\n"));
595 GNUNET_SCHEDULER_shutdown ();
599 GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL);
602 GNUNET_FS_file_information_destroy (fi, NULL, NULL);
603 GNUNET_SCHEDULER_shutdown ();
606 if (NULL == namespace)
609 priv = GNUNET_IDENTITY_ego_get_private_key (namespace);
610 pc = GNUNET_FS_publish_start (ctx, fi,
611 priv, this_id, next_id,
613 GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY :
614 GNUNET_FS_PUBLISH_OPTION_NONE);
619 _("Could not start publishing.\n"));
620 GNUNET_SCHEDULER_shutdown ();
628 * Function called by the directory scanner as we build the tree
629 * that we will need to publish later.
632 * @param filename which file we are making progress on
633 * @param is_directory #GNUNET_YES if this is a directory,
634 * #GNUNET_NO if this is a file
635 * #GNUNET_SYSERR if it is neither (or unknown)
636 * @param reason kind of progress we are making
639 directory_scan_cb (void *cls,
640 const char *filename,
642 enum GNUNET_FS_DirScannerProgressUpdateReason reason)
644 struct GNUNET_FS_ShareTreeItem *directory_scan_result;
648 case GNUNET_FS_DIRSCANNER_FILE_START:
651 if (is_directory == GNUNET_YES)
653 _("Scanning directory `%s'.\n"),
657 _("Scanning file `%s'.\n"),
661 case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
663 _("There was trouble processing file `%s', skipping it.\n"),
666 case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
670 _("Preprocessing complete.\n"));
672 case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
675 _("Extracting meta data from file `%s' complete.\n"),
678 case GNUNET_FS_DIRSCANNER_FINISHED:
682 _("Meta data extraction has finished.\n"));
683 directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
685 GNUNET_FS_share_tree_trim (directory_scan_result);
686 directory_trim_complete (directory_scan_result);
688 case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
691 _("Internal error scanning directory.\n"));
692 if (kill_task != NULL)
694 GNUNET_SCHEDULER_cancel (kill_task);
697 kill_task = GNUNET_SCHEDULER_add_now (&stop_scanner_task, NULL);
708 * Continuation proceeding with initialization after identity subsystem
709 * has been initialized.
711 * @param args0 filename to publish
714 identity_continuation (const char *args0)
719 if ( (NULL != pseudonym) &&
720 (NULL == namespace) )
723 _("Selected pseudonym `%s' unknown\n"),
725 GNUNET_SCHEDULER_shutdown ();
728 if (NULL != uri_string)
731 if (NULL == (uri = GNUNET_FS_uri_parse (uri_string, &emsg)))
734 _("Failed to parse URI: %s\n"),
737 GNUNET_SCHEDULER_shutdown ();
741 GNUNET_FS_publish_ksk (ctx, topKeywords,
744 GNUNET_FS_PUBLISH_OPTION_NONE,
745 &uri_ksk_continuation,
750 GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex))
752 if (0 != ACCESS (args0, R_OK))
755 _("Failed to access `%s': %s\n"),
758 GNUNET_free_non_null (ex);
761 ds = GNUNET_FS_directory_scan_start (args0,
764 &directory_scan_cb, NULL);
769 _("Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n"));
770 GNUNET_free_non_null (ex);
773 GNUNET_free_non_null (ex);
778 * Function called by identity service with known pseudonyms.
780 * @param cls closure with 'const char *' of filename to publish
781 * @param ego ego handle
782 * @param ctx context for application to store data for this ego
783 * (during the lifetime of this process, initially NULL)
784 * @param name name assigned by the user for this ego,
785 * NULL if the user just deleted the ego and it
786 * must thus no longer be used
789 identity_cb (void *cls,
790 struct GNUNET_IDENTITY_Ego *ego,
794 const char *args0 = cls;
798 identity_continuation (args0);
803 if (0 == strcmp (name, pseudonym))
809 * Main function that will be run by the scheduler.
812 * @param args remaining command-line arguments
813 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
814 * @param c configuration
820 const struct GNUNET_CONFIGURATION_Handle *c)
822 /* check arguments */
823 if ((NULL != uri_string) && (extract_only))
825 printf (_("Cannot extract metadata from a URI!\n"));
829 if (((NULL == uri_string) || (extract_only)) &&
830 ((NULL == args[0]) || (NULL != args[1])))
832 printf (_("You must specify one and only one filename for insertion.\n"));
836 if ((NULL != uri_string) && (NULL != args[0]))
838 printf (_("You must NOT specify an URI and a filename.\n"));
842 if (NULL != pseudonym)
846 FPRINTF (stderr, _("Option `%s' is required when using option `%s'.\n"),
853 { /* ordinary insertion checks */
857 _("Option `%s' makes no sense without option `%s'.\n"),
865 _("Option `%s' makes no sense without option `%s'.\n"),
873 GNUNET_FS_start (cfg, "gnunet-publish", &progress_cb, NULL,
874 GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
878 _("Could not initialize `%s' subsystem.\n"),
884 GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
887 if (NULL != pseudonym)
888 identity = GNUNET_IDENTITY_connect (cfg,
892 identity_continuation (args[0]);
897 * The main function to publish content to GNUnet.
899 * @param argc number of arguments from the command line
900 * @param argv command line arguments
901 * @return 0 ok, 1 on error
904 main (int argc, char *const *argv)
906 static const struct GNUNET_GETOPT_CommandLineOption options[] = {
907 {'a', "anonymity", "LEVEL",
908 gettext_noop ("set the desired LEVEL of sender-anonymity"),
909 1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level},
910 {'d', "disable-creation-time", NULL,
912 ("disable adding the creation time to the metadata of the uploaded file"),
913 0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
914 {'D', "disable-extractor", NULL,
915 gettext_noop ("do not use libextractor to add keywords or metadata"),
916 0, &GNUNET_GETOPT_set_one, &disable_extractor},
917 {'e', "extract", NULL,
919 ("print list of extracted keywords that would be used, but do not perform upload"),
920 0, &GNUNET_GETOPT_set_one, &extract_only},
921 {'k', "key", "KEYWORD",
923 ("add an additional keyword for the top-level file or directory"
924 " (this option can be specified multiple times)"),
925 1, &GNUNET_FS_getopt_set_keywords, &topKeywords},
926 {'m', "meta", "TYPE:VALUE",
927 gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
928 1, &GNUNET_FS_getopt_set_metadata, &meta},
929 {'n', "noindex", NULL,
930 gettext_noop ("do not index, perform full insertion (stores entire "
931 "file in encrypted form in GNUnet database)"),
932 0, &GNUNET_GETOPT_set_one, &do_insert},
935 ("specify ID of an updated version to be published in the future"
936 " (for namespace insertions only)"),
937 1, &GNUNET_GETOPT_set_string, &next_id},
938 {'p', "priority", "PRIORITY",
939 gettext_noop ("specify the priority of the content"),
940 1, &GNUNET_GETOPT_set_uint, &bo.content_priority},
941 {'P', "pseudonym", "NAME",
943 ("publish the files under the pseudonym NAME (place file into namespace)"),
944 1, &GNUNET_GETOPT_set_string, &pseudonym},
945 {'r', "replication", "LEVEL",
946 gettext_noop ("set the desired replication LEVEL"),
947 1, &GNUNET_GETOPT_set_uint, &bo.replication_level},
948 {'s', "simulate-only", NULL,
949 gettext_noop ("only simulate the process but do not do any "
950 "actual publishing (useful to compute URIs)"),
951 0, &GNUNET_GETOPT_set_one, &do_simulate},
953 gettext_noop ("set the ID of this version of the publication"
954 " (for namespace insertions only)"),
955 1, &GNUNET_GETOPT_set_string, &this_id},
957 gettext_noop ("URI to be published (can be used instead of passing a "
958 "file to add keywords to the file with the respective URI)"),
959 1, &GNUNET_GETOPT_set_string, &uri_string},
960 {'V', "verbose", NULL,
961 gettext_noop ("be verbose (print progress information)"),
962 0, &GNUNET_GETOPT_set_one, &verbose},
963 GNUNET_GETOPT_OPTION_END
966 GNUNET_TIME_year_to_time (GNUNET_TIME_get_current_year () + 2);
968 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
971 GNUNET_PROGRAM_run (argc, argv, "gnunet-publish [OPTIONS] FILENAME",
973 ("Publish a file or directory on GNUnet"),
974 options, &run, NULL)) ? ret : 1;
975 GNUNET_free ((void*) argv);
979 /* end of gnunet-publish.c */