2 This file is part of GNUnet.
3 Copyright (C) 2001-2013 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file fs/gnunet-publish.c
22 * @brief publishing files on GNUnet
23 * @author Christian Grothoff
24 * @author Krista Bennett
25 * @author James Blackwell
26 * @author Igor Wronsky
29 #include "gnunet_fs_service.h"
30 #include "gnunet_identity_service.h"
33 * Global return value from #main().
38 * Command line option 'verbose' set
40 static unsigned int verbose;
43 * Handle to our configuration.
45 static const struct GNUNET_CONFIGURATION_Handle *cfg;
48 * Handle for interaction with file-sharing service.
50 static struct GNUNET_FS_Handle *ctx;
53 * Handle to FS-publishing operation.
55 static struct GNUNET_FS_PublishContext *pc;
58 * Meta-data provided via command-line option.
60 static struct GNUNET_CONTAINER_MetaData *meta;
63 * Keywords provided via command-line option.
65 static struct GNUNET_FS_Uri *topKeywords;
68 * Options we set for published blocks.
70 static struct GNUNET_FS_BlockOptions bo = { { 0LL }, 1, 365, 1 };
73 * Value of URI provided on command-line (when not publishing
74 * a file but just creating UBlocks to refer to an existing URI).
76 static char *uri_string;
79 * Value of URI provided on command-line (when not publishing
80 * a file but just creating UBlocks to refer to an existing URI);
81 * parsed version of 'uri_string'.
83 static struct GNUNET_FS_Uri *uri;
86 * Command-line option for namespace publishing: identifier for updates
87 * to this publication.
92 * Command-line option for namespace publishing: identifier for this
98 * Command-line option identifying the pseudonym to use for the publication.
100 static char *pseudonym;
103 * Command-line option for 'inserting'
105 static int do_insert;
108 * Command-line option to disable meta data extraction.
110 static int disable_extractor;
113 * Command-line option to merely simulate publishing operation.
115 static int do_simulate;
118 * Command-line option to only perform meta data extraction, but not publish.
120 static int extract_only;
123 * Command-line option to disable adding creation time.
125 static int enable_creation_time;
128 * Handle to the directory scanner (for recursive insertions).
130 static struct GNUNET_FS_DirScanner *ds;
133 * Which namespace do we publish to? NULL if we do not publish to
136 static struct GNUNET_IDENTITY_Ego *namespace;
139 * Handle to identity service.
141 static struct GNUNET_IDENTITY_Handle *identity;
145 * We are finished with the publishing operation, clean up all
151 do_stop_task(void *cls)
153 struct GNUNET_FS_PublishContext *p;
157 GNUNET_FS_directory_scan_abort(ds);
160 if (NULL != identity)
162 GNUNET_IDENTITY_disconnect(identity);
169 GNUNET_FS_publish_stop(p);
178 GNUNET_CONTAINER_meta_data_destroy(meta);
183 GNUNET_FS_uri_destroy(uri);
190 * Called by FS client to give information about the progress of an
194 * @param info details about the event, specifying the event type
195 * and various bits about the event
196 * @return client-context (for the next progress call
197 * for this operation; should be set to NULL for
198 * SUSPEND and STOPPED events). The value returned
199 * will be passed to future callbacks in the respective
200 * field in the GNUNET_FS_ProgressInfo struct.
203 progress_cb(void *cls, const struct GNUNET_FS_ProgressInfo *info)
208 switch (info->status)
210 case GNUNET_FS_STATUS_PUBLISH_START:
213 case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
216 s = GNUNET_STRINGS_relative_time_to_string(info->value.publish.eta,
219 _("Publishing `%s' at %llu/%llu (%s remaining)\n"),
220 info->value.publish.filename,
221 (unsigned long long)info->value.publish.completed,
222 (unsigned long long)info->value.publish.size,
227 case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
230 s = GNUNET_STRINGS_relative_time_to_string(info->value.publish.specifics
231 .progress_directory.eta,
234 _("Publishing `%s' at %llu/%llu (%s remaining)\n"),
235 info->value.publish.filename,
237 info->value.publish.specifics.progress_directory.completed,
239 info->value.publish.specifics.progress_directory.total,
244 case GNUNET_FS_STATUS_PUBLISH_ERROR:
246 _("Error publishing: %s.\n"),
247 info->value.publish.specifics.error.message);
249 GNUNET_SCHEDULER_shutdown();
252 case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
254 _("Publishing `%s' done.\n"),
255 info->value.publish.filename);
257 GNUNET_FS_uri_to_string(info->value.publish.specifics.completed.chk_uri);
258 fprintf(stdout, _("URI is `%s'.\n"), suri);
260 if (NULL != info->value.publish.specifics.completed.sks_uri)
262 suri = GNUNET_FS_uri_to_string(
263 info->value.publish.specifics.completed.sks_uri);
264 fprintf(stdout, _("Namespace URI is `%s'.\n"), suri);
267 if (NULL == info->value.publish.pctx)
270 GNUNET_SCHEDULER_shutdown();
274 case GNUNET_FS_STATUS_PUBLISH_STOPPED:
275 GNUNET_break(NULL == pc);
278 case GNUNET_FS_STATUS_UNINDEX_START:
279 fprintf(stderr, "%s", _("Starting cleanup after abort\n"));
282 case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
285 case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
286 fprintf(stderr, "%s", _("Cleanup after abort completed.\n"));
287 GNUNET_FS_unindex_stop(info->value.unindex.uc);
290 case GNUNET_FS_STATUS_UNINDEX_ERROR:
291 fprintf(stderr, "%s", _("Cleanup after abort failed.\n"));
292 GNUNET_FS_unindex_stop(info->value.unindex.uc);
295 case GNUNET_FS_STATUS_UNINDEX_STOPPED:
299 fprintf(stderr, _("Unexpected status: %d\n"), info->status);
302 return ""; /* non-null */
307 * Print metadata entries (except binary
308 * metadata and the filename).
311 * @param plugin_name name of the plugin that generated the meta data
312 * @param type type of the meta data
313 * @param format format of data
314 * @param data_mime_type mime type of @a data
315 * @param data value of the meta data
316 * @param data_size number of bytes in @a data
320 meta_printer(void *cls,
321 const char *plugin_name,
322 enum EXTRACTOR_MetaType type,
323 enum EXTRACTOR_MetaFormat format,
324 const char *data_mime_type,
328 if ((EXTRACTOR_METAFORMAT_UTF8 != format) &&
329 (EXTRACTOR_METAFORMAT_C_STRING != format))
331 if (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type)
333 #if HAVE_LIBEXTRACTOR
334 fprintf(stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string(type), data);
336 fprintf(stdout, "\t%d - %s\n", type, data);
343 * Iterator printing keywords
346 * @param keyword the keyword
347 * @param is_mandatory is the keyword mandatory (in a search)
348 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to abort
351 keyword_printer(void *cls, const char *keyword, int is_mandatory)
353 fprintf(stdout, "\t%s\n", keyword);
359 * Function called on all entries before the publication. This is
360 * where we perform modifications to the default based on command-line
364 * @param fi the entry in the publish-structure
365 * @param length length of the file or directory
366 * @param m metadata for the file or directory (can be modified)
367 * @param uri pointer to the keywords that will be used for this entry (can be modified)
368 * @param bo block options
369 * @param do_index should we index?
370 * @param client_info pointer to client context set upon creation (can be modified)
371 * @return #GNUNET_OK to continue, #GNUNET_NO to remove
372 * this entry from the directory, #GNUNET_SYSERR
373 * to abort the iteration
376 publish_inspector(void *cls,
377 struct GNUNET_FS_FileInformation *fi,
379 struct GNUNET_CONTAINER_MetaData *m,
380 struct GNUNET_FS_Uri **uri,
381 struct GNUNET_FS_BlockOptions *bo,
387 struct GNUNET_FS_Uri *new_uri;
391 if ((disable_extractor) && (NULL != *uri))
393 GNUNET_FS_uri_destroy(*uri);
396 if (NULL != topKeywords)
400 new_uri = GNUNET_FS_uri_ksk_merge(topKeywords, *uri);
401 GNUNET_FS_uri_destroy(*uri);
403 GNUNET_FS_uri_destroy(topKeywords);
413 GNUNET_CONTAINER_meta_data_merge(m, meta);
414 GNUNET_CONTAINER_meta_data_destroy(meta);
417 if (enable_creation_time)
418 GNUNET_CONTAINER_meta_data_add_publication_date(m);
421 fn = GNUNET_CONTAINER_meta_data_get_by_type(
423 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
424 fs = GNUNET_STRINGS_byte_size_fancy(length);
425 fprintf(stdout, _("Meta data for file `%s' (%s)\n"), fn, fs);
426 GNUNET_CONTAINER_meta_data_iterate(m, &meta_printer, NULL);
427 fprintf(stdout, _("Keywords for file `%s' (%s)\n"), fn, fs);
431 GNUNET_FS_uri_ksk_get_keywords(*uri, &keyword_printer, NULL);
432 fprintf(stdout, "%s", "\n");
434 if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory(m))
435 GNUNET_FS_file_information_inspect(fi, &publish_inspector, fi);
441 * Function called upon completion of the publishing
442 * of the UBLOCK for the SKS URI. As this is the last
443 * step, stop our interaction with FS (clean up).
445 * @param cls NULL (closure)
446 * @param sks_uri URI for the block that was published
447 * @param emsg error message, NULL on success
450 uri_sks_continuation(void *cls,
451 const struct GNUNET_FS_Uri *sks_uri,
456 fprintf(stderr, "%s\n", emsg);
459 GNUNET_SCHEDULER_shutdown();
464 * Function called upon completion of the publishing
465 * of the UBLOCK for the KSK URI. Continue with
466 * publishing the SKS URI (if applicable) or clean up.
468 * @param cls NULL (closure)
469 * @param ksk_uri URI for the block that was published
470 * @param emsg error message, NULL on success
473 uri_ksk_continuation(void *cls,
474 const struct GNUNET_FS_Uri *ksk_uri,
477 const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;
481 fprintf(stderr, "%s\n", emsg);
484 if (NULL == namespace)
486 GNUNET_SCHEDULER_shutdown();
489 priv = GNUNET_IDENTITY_ego_get_private_key(namespace);
490 GNUNET_FS_publish_sks(ctx,
497 GNUNET_FS_PUBLISH_OPTION_NONE,
498 &uri_sks_continuation,
504 * Iterate over the results from the directory scan and extract
505 * the desired information for the publishing operation.
507 * @param item root with the data from the directroy scan
508 * @return handle with the information for the publishing operation
510 static struct GNUNET_FS_FileInformation *
511 get_file_information(struct GNUNET_FS_ShareTreeItem *item)
513 struct GNUNET_FS_FileInformation *fi;
514 struct GNUNET_FS_FileInformation *fic;
515 struct GNUNET_FS_ShareTreeItem *child;
517 if (GNUNET_YES == item->is_directory)
519 if (NULL == item->meta)
520 item->meta = GNUNET_CONTAINER_meta_data_create();
521 GNUNET_CONTAINER_meta_data_delete(item->meta,
522 EXTRACTOR_METATYPE_MIMETYPE,
525 GNUNET_FS_meta_data_make_directory(item->meta);
526 if (NULL == item->ksk_uri)
528 const char *mime = GNUNET_FS_DIRECTORY_MIME;
529 item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args(1, &mime);
532 GNUNET_FS_uri_ksk_add_keyword(item->ksk_uri,
533 GNUNET_FS_DIRECTORY_MIME,
535 fi = GNUNET_FS_file_information_create_empty_directory(ctx,
541 for (child = item->children_head; child; child = child->next)
543 fic = get_file_information(child);
544 GNUNET_break(GNUNET_OK == GNUNET_FS_file_information_add(fi, fic));
549 fi = GNUNET_FS_file_information_create_from_file(ctx,
562 * We've finished scanning the directory and optimized the meta data.
563 * Begin the publication process.
565 * @param directory_scan_result result from the directory scan, freed in this function
568 directory_trim_complete(struct GNUNET_FS_ShareTreeItem *directory_scan_result)
570 struct GNUNET_FS_FileInformation *fi;
571 const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;
573 fi = get_file_information(directory_scan_result);
574 GNUNET_FS_share_tree_free(directory_scan_result);
577 fprintf(stderr, "%s", _("Could not publish\n"));
579 GNUNET_SCHEDULER_shutdown();
582 GNUNET_FS_file_information_inspect(fi, &publish_inspector, NULL);
585 GNUNET_FS_file_information_destroy(fi, NULL, NULL);
586 GNUNET_SCHEDULER_shutdown();
589 if (NULL == namespace)
592 priv = GNUNET_IDENTITY_ego_get_private_key(namespace);
593 pc = GNUNET_FS_publish_start(ctx,
599 ? GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY
600 : GNUNET_FS_PUBLISH_OPTION_NONE);
603 fprintf(stderr, "%s", _("Could not start publishing.\n"));
605 GNUNET_SCHEDULER_shutdown();
612 * Function called by the directory scanner as we build the tree
613 * that we will need to publish later.
616 * @param filename which file we are making progress on
617 * @param is_directory #GNUNET_YES if this is a directory,
618 * #GNUNET_NO if this is a file
619 * #GNUNET_SYSERR if it is neither (or unknown)
620 * @param reason kind of progress we are making
623 directory_scan_cb(void *cls,
624 const char *filename,
626 enum GNUNET_FS_DirScannerProgressUpdateReason reason)
628 struct GNUNET_FS_ShareTreeItem *directory_scan_result;
632 case GNUNET_FS_DIRSCANNER_FILE_START:
635 if (is_directory == GNUNET_YES)
636 fprintf(stdout, _("Scanning directory `%s'.\n"), filename);
638 fprintf(stdout, _("Scanning file `%s'.\n"), filename);
642 case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
644 _("There was trouble processing file `%s', skipping it.\n"),
648 case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
650 fprintf(stdout, "%s", _("Preprocessing complete.\n"));
653 case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
656 _("Extracting meta data from file `%s' complete.\n"),
660 case GNUNET_FS_DIRSCANNER_FINISHED:
662 fprintf(stdout, "%s", _("Meta data extraction has finished.\n"));
663 directory_scan_result = GNUNET_FS_directory_scan_get_result(ds);
665 GNUNET_FS_share_tree_trim(directory_scan_result);
666 directory_trim_complete(directory_scan_result);
669 case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
670 fprintf(stdout, "%s", _("Error scanning directory.\n"));
672 GNUNET_SCHEDULER_shutdown();
684 * Continuation proceeding with initialization after identity subsystem
685 * has been initialized.
687 * @param args0 filename to publish
690 identity_continuation(const char *args0)
695 if ((NULL != pseudonym) && (NULL == namespace))
697 fprintf(stderr, _("Selected pseudonym `%s' unknown\n"), pseudonym);
699 GNUNET_SCHEDULER_shutdown();
702 if (NULL != uri_string)
705 if (NULL == (uri = GNUNET_FS_uri_parse(uri_string, &emsg)))
707 fprintf(stderr, _("Failed to parse URI: %s\n"), emsg);
710 GNUNET_SCHEDULER_shutdown();
713 GNUNET_FS_publish_ksk(ctx,
718 GNUNET_FS_PUBLISH_OPTION_NONE,
719 &uri_ksk_continuation,
724 GNUNET_CONFIGURATION_get_value_string(cfg, "FS", "EXTRACTORS", &ex))
726 if (0 != access(args0, R_OK))
729 _("Failed to access `%s': %s\n"),
732 GNUNET_free_non_null(ex);
735 ds = GNUNET_FS_directory_scan_start(args0,
746 "Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n"));
747 GNUNET_free_non_null(ex);
750 GNUNET_free_non_null(ex);
755 * Function called by identity service with known pseudonyms.
757 * @param cls closure with 'const char *' of filename to publish
758 * @param ego ego handle
759 * @param ctx context for application to store data for this ego
760 * (during the lifetime of this process, initially NULL)
761 * @param name name assigned by the user for this ego,
762 * NULL if the user just deleted the ego and it
763 * must thus no longer be used
766 identity_cb(void *cls,
767 struct GNUNET_IDENTITY_Ego *ego,
771 const char *args0 = cls;
775 identity_continuation(args0);
780 if (0 == strcmp(name, pseudonym))
786 * Main function that will be run by the scheduler.
789 * @param args remaining command-line arguments
790 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
791 * @param c configuration
797 const struct GNUNET_CONFIGURATION_Handle *c)
799 /* check arguments */
800 if ((NULL != uri_string) && (extract_only))
802 printf(_("Cannot extract metadata from a URI!\n"));
806 if (((NULL == uri_string) || (extract_only)) &&
807 ((NULL == args[0]) || (NULL != args[1])))
809 printf(_("You must specify one and only one filename for insertion.\n"));
813 if ((NULL != uri_string) && (NULL != args[0]))
815 printf(_("You must NOT specify an URI and a filename.\n"));
819 if (NULL != pseudonym)
824 _("Option `%s' is required when using option `%s'.\n"),
832 { /* ordinary insertion checks */
836 _("Option `%s' makes no sense without option `%s'.\n"),
845 _("Option `%s' makes no sense without option `%s'.\n"),
853 ctx = GNUNET_FS_start(cfg,
857 GNUNET_FS_FLAGS_NONE,
858 GNUNET_FS_OPTIONS_END);
861 fprintf(stderr, _("Could not initialize `%s' subsystem.\n"), "FS");
865 GNUNET_SCHEDULER_add_shutdown(&do_stop_task, NULL);
866 if (NULL != pseudonym)
867 identity = GNUNET_IDENTITY_connect(cfg, &identity_cb, args[0]);
869 identity_continuation(args[0]);
874 * The main function to publish content to GNUnet.
876 * @param argc number of arguments from the command line
877 * @param argv command line arguments
878 * @return 0 ok, 1 on error
881 main(int argc, char *const *argv)
883 struct GNUNET_GETOPT_CommandLineOption options[] =
884 { GNUNET_GETOPT_option_uint('a',
888 "set the desired LEVEL of sender-anonymity"),
889 &bo.anonymity_level),
890 GNUNET_GETOPT_option_flag(
893 gettext_noop("do not use libextractor to add keywords or metadata"),
895 GNUNET_GETOPT_option_flag('E',
896 "enable-creation-time",
898 "enable adding the creation time to the "
899 "metadata of the uploaded file"),
900 &enable_creation_time),
901 GNUNET_GETOPT_option_flag('e',
904 "print list of extracted keywords that would "
905 "be used, but do not perform upload"),
907 GNUNET_FS_GETOPT_KEYWORDS(
912 "add an additional keyword for the top-level "
913 "file or directory (this option can be specified multiple times)"),
915 GNUNET_FS_GETOPT_METADATA(
919 gettext_noop("set the meta-data for the given TYPE to the given VALUE"),
921 GNUNET_GETOPT_option_flag(
924 gettext_noop("do not index, perform full insertion (stores "
925 "entire file in encrypted form in GNUnet database)"),
927 GNUNET_GETOPT_option_string(
931 gettext_noop("specify ID of an updated version to be "
932 "published in the future (for namespace insertions only)"),
934 GNUNET_GETOPT_option_uint('p',
938 "specify the priority of the content"),
939 &bo.content_priority),
940 GNUNET_GETOPT_option_string('P',
944 "publish the files under the pseudonym "
945 "NAME (place file into namespace)"),
947 GNUNET_GETOPT_option_uint('r',
951 "set the desired replication LEVEL"),
952 &bo.replication_level),
953 GNUNET_GETOPT_option_flag('s',
956 "only simulate the process but do not do "
957 "any actual publishing (useful to compute URIs)"),
959 GNUNET_GETOPT_option_string('t',
963 "set the ID of this version of the publication "
964 "(for namespace insertions only)"),
966 GNUNET_GETOPT_option_string(
971 "URI to be published (can be used instead of passing a "
972 "file to add keywords to the file with the respective URI)"),
975 GNUNET_GETOPT_option_verbose(&verbose),
977 GNUNET_GETOPT_OPTION_END };
980 GNUNET_TIME_year_to_time(GNUNET_TIME_get_current_year() + 2);
982 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args(argc, argv, &argc, &argv))
986 GNUNET_PROGRAM_run(argc,
988 "gnunet-publish [OPTIONS] FILENAME",
989 gettext_noop("Publish a file or directory on GNUnet"),
995 GNUNET_free((void *)argv);
999 /* end of gnunet-publish.c */