2 This file is part of GNUnet
3 (C) 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * directory creation/inspection API
22 * unindex start/stop API
23 * resume notifications
24 * ProgressCallback: struct/union instead of tons of args?
25 * download options (no temporary files -- what about no files at all?)
28 * @file include/gnunet_fs_service.h
29 * @brief support for file-sharing via GNUnet
30 * @author Christian Grothoff
33 #ifndef GNUNET_FS_LIB_H
34 #define GNUNET_FS_LIB_H
36 #include "gnunet_util_lib.h"
41 #if 0 /* keep Emacsens' auto-indent happy */
47 * Version number of the implementation.
50 * 1.x.x: initial version with triple GNUNET_hash and merkle tree
51 * 2.x.x: root node with mime-type, filename and version number
52 * 2.1.x: combined GNUNET_EC_ContentHashKey/3HASH encoding with 25:1 super-nodes
53 * 2.2.x: with directories
54 * 3.0.x: with namespaces
55 * 3.1.x: with namespace meta-data
56 * 3.2.x: with collections
57 * 4.0.x: with expiration, variable meta-data, kblocks
58 * 4.1.x: with new error and configuration handling
59 * 5.0.x: with location URIs
60 * 6.0.0: with support for OR in KSKs
61 * 6.1.x: with simplified namespace support
62 * 9.0.0: CPS-style integrated API
64 #define GNUNET_FS_VERSION 0x00090000
66 #define GNUNET_DIRECTORY_MIME "application/gnunet-directory"
67 #define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
68 #define GNUNET_DIRECTORY_EXT ".gnd"
72 #define GNUNET_FS_URI_PREFIX "gnunet://fs/"
73 #define GNUNET_FS_SEARCH_INFIX "ksk/"
74 #define GNUNET_FS_SUBSPACE_INFIX "sks/"
75 #define GNUNET_FS_FILE_INFIX "chk/"
76 #define GNUNET_FS_LOCATION_INFIX "loc/"
79 * Iterator over keywords
82 * @param keyword the keyword
83 * @param is_mandatory is the keyword mandatory (in a search)
84 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
86 typedef int (*GNUNET_FS_KeywordIterator) (void *cls,
91 * Does the meta-data claim that this is a directory?
92 * Checks if the mime-type is that of a GNUnet directory.
94 * @return GNUNET_YES if it is, GNUNET_NO if it is not, GNUNET_SYSERR if
95 * we have no mime-type information (treat as 'GNUNET_NO')
97 int GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData *md);
101 * A URI (in internal representation).
103 struct GNUNET_FS_Uri;
106 * Get a unique key from a URI. This is for putting URIs
107 * into HashMaps. The key may change between FS implementations.
109 void GNUNET_FS_uri_to_key (const struct GNUNET_FS_Uri *uri,
110 GNUNET_HashCode * key);
113 * Convert a URI to a UTF-8 String.
115 char *GNUNET_FS_uri_to_string (const struct GNUNET_FS_Uri *uri);
118 * Convert keyword URI to a human readable format
119 * (i.e. the search query that was used in the first place)
121 char *GNUNET_FS_ksk_uri_ksk_to_string_fancy (const struct
125 * Convert a UTF-8 String to a URI.
127 * @param uri string to parse
128 * @param emsg where to store the parser error message (if any)
129 * @return NULL on error
131 struct GNUNET_FS_Uri *GNUNET_FS_uri_parse (
132 const char *uri, char **emsg);
137 void GNUNET_FS_uri_destroy (struct GNUNET_FS_Uri *uri);
140 * How many keywords are ANDed in this keyword URI?
141 * @return 0 if this is not a keyword URI
143 unsigned int GNUNET_FS_uri_ksk_get_keyword_count (const struct
148 * Iterate over all keywords in this keyword URI.
150 * @return -1 if this is not a keyword URI, otherwise number of
151 * keywords iterated over until iterator aborted
153 int GNUNET_FS_uri_ksk_get_keywords (const struct GNUNET_FS_Uri *uri,
154 GNUNET_FS_KeywordIterator
155 iterator, void *iterator_cls);
158 * Obtain the identity of the peer offering the data
159 * @return GNUNET_SYSERR if this is not a location URI, otherwise GNUNET_OK
161 int GNUNET_FS_uri_loc_get_peer_identity (const struct GNUNET_FS_Uri
163 struct GNUNET_PeerIdentity * peer);
166 * Obtain the URI of the content itself.
168 * @return NULL if argument is not a location URI
170 struct GNUNET_FS_Uri *GNUNET_FS_uri_loc_get_uri (const struct
175 * Construct a location URI.
177 * @param baseURI content offered by the sender
178 * @param expiration_time how long will the content be offered?
179 * @return the location URI, NULL on error
181 struct GNUNET_FS_Uri *GNUNET_FS_uri_loc_create (const struct
182 GNUNET_FS_Uri *baseUri,
183 struct GNUNET_CONFIGURATION_Handle *cfg,
184 struct GNUNET_TIME_Absolute
192 struct GNUNET_FS_Uri *GNUNET_FS_uri_dup (const struct
196 * Create an FS URI from a single user-supplied string of keywords.
197 * The string is broken up at spaces into individual keywords.
198 * Keywords that start with "+" are mandatory. Double-quotes can
199 * be used to prevent breaking up strings at spaces (and also
200 * to specify non-mandatory keywords starting with "+").
202 * Keywords must contain a balanced number of double quotes and
203 * double quotes can not be used in the actual keywords (for
204 * example, the string '""foo bar""' will be turned into two
205 * "OR"ed keywords 'foo' and 'bar', not into '"foo bar"'.
207 * @return an FS URI for the given keywords, NULL
208 * if keywords is not legal (i.e. empty).
210 struct GNUNET_FS_Uri *GNUNET_FS_uri_ksk_create (
215 * Create an FS URI from a user-supplied command line of keywords.
216 * Arguments should start with "+" to indicate mandatory
219 * @param argc number of keywords
220 * @param argv keywords (double quotes are not required for
221 * keywords containing spaces; however, double
222 * quotes are required for keywords starting with
223 * "+"); there is no mechanism for having double
224 * quotes in the actual keywords (if the user
225 * did specifically specify double quotes, the
226 * caller should convert each double quote
227 * into two single quotes).
228 * @return an FS URI for the given keywords, NULL
229 * if keywords is not legal (i.e. empty).
231 struct GNUNET_FS_Uri *GNUNET_FS_uri_ksk_create_from_args (
238 * Test if two URIs are equal.
240 int GNUNET_FS_uri_test_equal (const struct GNUNET_FS_Uri *u1,
241 const struct GNUNET_FS_Uri *u2);
244 * Is this a namespace URI?
246 int GNUNET_FS_uri_test_sks (const struct GNUNET_FS_Uri *uri);
249 * Get the ID of a namespace from the given
252 int GNUNET_FS_uri_sks_get_namespace (const struct GNUNET_FS_Uri *uri,
253 GNUNET_HashCode * nsid);
256 * Get the content identifier of an SKS URI.
258 * @return NULL on error
260 char *GNUNET_FS_uri_sks_get_content_id (const struct GNUNET_FS_Uri
265 * Is this a keyword URI?
267 int GNUNET_FS_uri_test_ksk (const struct GNUNET_FS_Uri *uri);
270 * Is this a file (or directory) URI?
272 int GNUNET_FS_uri_test_chk (const struct GNUNET_FS_Uri *uri);
275 * What is the size of the file that this URI
278 uint64_t GNUNET_FS_uri_chk_get_file_size (const struct GNUNET_FS_Uri
282 * Is this a location URI?
284 int GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri);
289 * Construct a keyword-URI from meta-data (take all entries
290 * in the meta-data and construct one large keyword URI
291 * that lists all keywords that can be found in the meta-data).
293 struct GNUNET_FS_Uri *GNUNET_FS_uri_ksk_create_from_meta_data (const struct
294 GNUNET_MetaData *md);
297 * @param scls must be of type "struct GNUNET_FS_Uri **"
300 GNUNET_FS_getopt_configure_set_keywords (GNUNET_GETOPT_CommandLineProcessorContext
306 * @param scls must be of type "struct GNUNET_MetaData **"
309 GNUNET_FS_getopt_configure_set_metadata (GNUNET_GETOPT_CommandLineProcessorContext
316 /* ************************* sharing API ***************** */
320 * Possible status codes used in the callback for the
321 * various file-sharing operations. On each file (or search),
322 * the callback is guaranteed to be called once with "START"
323 * and once with STOPPED; calls with PROGRESS, ERROR or COMPLETED
324 * are optional and depend on the circumstances; parent operations
325 * will be STARTED before child-operations and STOPPED after
326 * their respective child-operations. START and STOP signals
327 * are typically generated either due to explicit client requests
328 * or because of suspend/resume operations.
330 enum GNUNET_FS_Status
332 GNUNET_FS_STATUS_UPLOAD_START,
333 GNUNET_FS_STATUS_UPLOAD_PROGRESS,
334 GNUENT_FS_STATUS_UPLOAD_ERROR,
335 GNUNET_FS_STATUS_UPLOAD_COMPLETED,
336 GNUNET_FS_STATUS_UPLOAD_STOPPED,
337 GNUNET_FS_DOWNLOAD_START,
338 GNUNET_FS_DOWNLOAD_PROGRESS,
339 GNUNET_FS_DOWNLOAD_ERROR,
340 GNUNET_FS_DOWNLOAD_COMPLETED,
341 GNUNET_FS_DOWNLOAD_STOPPED,
342 GNUNET_FS_SEARCH_START,
343 GNUNET_FS_SEARCH_PROGRESS,
344 GNUNET_FS_SEARCH_ERROR,
345 GNUNET_FS_SEARCH_STOPPED
346 /* fixme: unindex status codes... */
351 * Notification of FS to a client about the progress of an
352 * operation. Callbacks of this type will be used for uploads,
353 * downloads and searches. Some of the arguments depend a bit
354 * in their meaning on the context in which the callback is used.
357 * @param ctx location where the callback can store a context pointer
358 * to keep track of things for this specific operation
359 * @param pctx context pointer set by the callback for the parent operation
360 * (NULL if there is no parent operation); for a search result,
361 * the actual search is the parent and the individual search results
362 * are the children (multiple calls for the same search result can
363 * be used whenever availability/certainty or metadata values change)
364 * @param filename name of the file that this update is about, NULL for
366 * @param status specific status code
367 * @param availability value between 0 and 100 indicating how likely
368 * we think it is that this search result is actually available
369 * in the network (or, in the case of a download, that the download
370 * will complete); always 100 for uploads; percentage of blocks
371 * that could be unindexed so far for unindexing operations
372 * (indicates how many blocks in the indexed file changed in
374 * @param certainty how certain are we that the availability value is
375 * actually correct? certainty is also between 0 and 100.
376 * @param fsize number of bytes that will need to be processed (for this file)
377 * @param completed number of bytes that have been processed (for this file)
378 * @param offset offset of the data of buffer in the file
379 * @param eta absolute estimated time for the completion of the operation
380 * @param uri pointer to CHK URI for search results and downloads; pointer
381 * to KSK uri for uploads; client can modify KSK uri to change the
382 * set of keywords that will be used
383 * @param meta metadata for search results and downloads (NULL for downloads
384 * if no metadata is available); can be modified for uploads to change
385 * metadata that will be used
386 * @param bsize number of bytes in the buffer
387 * @param buffer pointer to the last bytes processed; will be a plaintext
388 * buffer for files (with content downloaded or uploaded) and
389 * NULL when searching; points to an error message of bsize bytes
390 * if this callback is used to signal an error
391 * @return GNUNET_SYSERR to abort the overall operation; GNUNET_NO to
392 * stop this specific operation (do not share this file or skip
393 * this download; GNUNET_NO has no meaning for search results);
394 * GNUNET_YES to continue processing as usual
396 typedef int (*GNUNET_FS_ProgressCallback)
400 const char *filename,
401 enum GNUNET_FS_Status status,
406 uint64_t offset, struct GNUNET_TIME_Absolute eta,
407 struct GNUNET_FS_Uri **uri,
408 struct GNUNET_CONTAINER_MetaData *meta,
409 size_t bsize, const void *buffer);
413 * Handle to the file-sharing service.
415 struct GNUNET_FS_Handle;
419 * Setup a connection to the file-sharing service.
421 * @param client_name unique identifier for this client
423 struct GNUNET_FS_Handle *
424 GNUNET_FS_start (struct GNUNET_SCHEDULER_Handle *sched,
425 struct GNUNET_CONFIGURATION_Handle *cfg,
426 const char *client_name,
427 GNUNET_FS_ProgressCallback upcb,
431 * Close our connection with the file-sharing service.
432 * The callback given to GNUNET_FS_start will no longer be
433 * called after this function returns.
435 void GNUNET_FS_stop (struct GNUNET_FS_Handle *h);
439 * Handle to one of our namespaces.
441 struct GNUNET_FS_Namespace;
443 struct GNUNET_FS_ShareContext;
446 * Share a file or directory.
448 * @param ctx initial value to use for the '*ctx' in the callback
449 * @param priority what is the priority for OUR node to
450 * keep this file available? Use 0 for maximum anonymity and
451 * minimum reliability...
452 * @param do_index GNUNET_YES for index, GNUNET_NO for insertion,
453 * GNUNET_SYSERR for simulation
454 * @param namespace namespace to share the file in, NULL for no namespace
455 * @param nid identifier to use for the shared content in the namespace
456 * (can be NULL, must be NULL if namespace is NULL)
457 * @param nuid update-identifier that will be used for future updates
458 * (can be NULL, must be NULL if namespace or nid is NULL)
460 struct GNUNET_FS_ShareContext *
461 GNUNET_FS_share_start (struct GNUNET_FS_Handle *h,
463 const char *filename,
465 unsigned int anonymity,
466 unsigned int priority,
467 struct GNUNET_TIME_Absolute expirationTime,
468 struct GNUNET_FS_Namespace *namespace
469 const char *nid, const char *nuid);
471 void GNUNET_FS_share_stop (struct GNUNET_FS_ShareContext *sc);
474 * @return GNUNET_OK to continue iteration, GNUNET_SYSERR to abort
476 typedef int (*GNUNET_FS_FileProcessor) (void *cls, const char *filename);
480 * Iterate over all indexed files.
482 void GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *sched,
483 GNUNET_FS_FileProcessor iterator,
484 void *iterator_closure);
489 void GNUNET_FS_unindex (struct GNUNET_FS_Handle *h,
490 const char *filename,
495 * Create a new namespace (and publish an advertismement).
496 * This publishes both an GNUNET_EC_NBlock in the namespace itself
497 * as well as KNBlocks under all keywords specified in
498 * the advertisementURI.
500 * @param anonymity for the namespace advertismement
501 * @param priority for the namespace advertisement
502 * @param expiration for the namespace advertisement
503 * @param advertisementURI the keyword (!) URI to advertise the
504 * namespace under (GNUNET_EC_KNBlock)
505 * @param meta meta-data for the namespace advertisement
506 * (will be used to derive a name)
507 * @param rootEntry name of the root entry in the namespace (for
508 * the namespace advertisement)
510 * @return uri of the advertisement
512 struct GNUNET_FS_Uri *
513 GNUNET_FS_namespace_advertise (struct GNUNET_FS_Namespace *namespace,
519 unsigned int priority,
520 struct GNUNET_TIME_Absolute
525 const char *rootEntry);
528 * Create a namespace with the given name; if one already
529 * exists, return a handle to the existing namespace.
531 * @return handle to the namespace, NULL on error
533 struct GNUNET_FS_Namespace *
534 GNUNET_FS_namespace_create (struct GNUNET_FS_Handle *h,
539 * Delete a namespace handle.
541 * @param freeze prevents future insertions; creating a namespace
542 * with the same name again will create a fresh namespace instead
544 * @return GNUNET_OK on success, GNUNET_SYSERR on error
546 int GNUNET_FS_namespace_delete (struct GNUNET_FS_Namespace *namespace,
551 * Callback with information about local (!) namespaces.
552 * Contains the names of the local namespace and the global
555 typedef void (*GNUNET_FS_NamespaceInfoProcessor) (void *cls,
558 const GNUNET_HashCode *id);
561 * Build a list of all available local (!) namespaces
562 * The returned names are only the nicknames since
563 * we only iterate over the local namespaces.
565 * @param list where to store the names (is allocated, caller frees)
566 * @return GNUNET_SYSERR on error, otherwise the number of pseudonyms in list
568 int GNUNET_FS_namespace_list (struct GNUNET_FS_Handle *h,
569 GNUNET_FS_NamespaceProcessor cb,
572 typedef void (*GNUNET_FS_IdentifierProcessor)(void *cls,
573 const char *last_id, const struct GNUNET_FS_Uri *last_uri,
574 const struct GNUNET_CONTAINER_MetaData *last_meta,
575 const char *next_id);
578 * List all of the identifiers in the namespace for
579 * which we could produce an update.
583 GNUNET_FS_namespace_list_updateable (struct
584 GNUNET_FS_Namespace *namespace,
585 GNUNET_FS_IdentifierProcessor ip,
589 struct GNUNET_FS_SearchContext;
592 * Start search for content.
594 * @param uri specifies the search parameters;
595 * this must be a simple URI (with a single
598 struct GNUNET_FS_SearchContext *
599 GNUNET_FS_search_start (struct
609 void GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc);
611 void GNUNET_FS_search_resume (struct GNUNET_FS_SearchContext *sc);
614 * Stop search for content.
616 * @param uri specifies the search parameters
617 * @param uri set to the URI of the uploaded file
619 void GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sctx);
622 struct GNUNET_FS_DownloadContext;
625 * Download parts of a file. Note that this will store
626 * the blocks at the respective offset in the given file. Also, the
627 * download is still using the blocking of the underlying FS
628 * encoding. As a result, the download may *write* outside of the
629 * given boundaries (if offset and length do not match the 32k FS
630 * block boundaries). <p>
632 * This function should be used to focus a download towards a
633 * particular portion of the file (optimization), not to strictly
634 * limit the download to exactly those bytes.
636 * @param uri the URI of the file (determines what to download)
637 * @param filename where to store the file, maybe NULL (then no file is
639 * @param no_temporaries set to GNUNET_YES to disallow generation of temporary files
640 * @param start starting offset
641 * @param length length of the download (starting at offset)
643 struct GNUNET_FS_DownloadContext
644 *GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
645 const struct GNUNET_FS_Uri *uri,
646 const char *filename,
647 unsigned long long offset,
648 unsigned long long length,
649 unsigned int anonymity,
652 struct GNUNET_FS_DownloadContext *parent);
655 * Stop a download (aborts if download is incomplete).
656 * @param do_delete delete files of incomplete downloads
659 GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext
665 * Iterate over all entries in a directory. Note that directories
666 * are structured such that it is possible to iterate over the
667 * individual blocks as well as over the entire directory. Thus
668 * a client can call this function on the buffer in the
669 * GNUNET_FS_ProgressCallback.
671 * @param data pointer to the beginning of the directory
672 * @param size number of bytes in data
673 * @param offset offset of data in the file
675 void GNUNET_FS_directory_list_contents (size_t size,
678 GNUNET_FS_SearchResultProcessor
679 spcb, void *spcbClosure);
682 * Create a directory.
684 * @param data pointer set to the beginning of the directory
685 * @param len set to number of bytes in data
686 * @param count number of entries in uris and metaDatas
687 * @param uris URIs of the files in the directory
688 * @param metaDatas meta-data for the files (must match
689 * respective values at same offset in in uris)
690 * @param meta meta-data for the directory. The meta entry
691 * is extended with the mime-type for a GNUnet directory.
692 * @return GNUNET_OK on success, GNUNET_SYSERR on error
694 int GNUNET_FS_directory_create (
696 unsigned long long *len,
698 const GNUNET_FS_FileInfo * fis,
699 struct GNUNET_MetaData *meta);
705 * Initialize collection.
707 void GNUNET_FS_collection_start (struct GNUNET_FS_Handle *h,
708 struct GNUNET_FS_Namespace *namespace);
713 * @return GNUNET_OK on success, GNUNET_SYSERR if no collection is active
715 int GNUNET_CO_collection_stop (struct GNUNET_FS_Handle *h);
719 * Are we using a collection?
721 * @return NULL if there is no collection,
723 struct GNUNET_FS_Namespace *GNUNET_FS_collection_get(struct GNUNET_FS_Handle *h);
726 * Publish an update of the current collection information to the
727 * network now. The function has no effect if the collection has not
728 * changed since the last publication. If we are currently not
729 * collecting, this function does nothing.
731 void GNUNET_FS_collection_publish (struct GNUNET_FS_Handle *h);
734 * If we are currently building a collection, publish the given file
735 * information in that collection. If we are currently not
736 * collecting, this function does nothing.
738 void GNUNET_FS_collection_add (const struct GNUNET_FS_Handle *h,
739 const struct GNUNET_FS_Uri *uri,
740 const struct GNUNET_CONTAINER_MetaData *meta);
745 * Convert namespace URI to a human readable format
746 * (using the namespace description, if available).
748 char *GNUNET_FS_uri_sks_to_string_fancy (
750 GNUNET_CONFIGURATION_Handle *cfg,
751 const struct GNUNET_FS_Uri
756 #if 0 /* keep Emacsens' auto-indent happy */