2 This file is part of GNUnet.
3 (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
23 * @brief shared definitions for the FS library
24 * @author Igor Wronsky, Christian Grothoff
29 #include "gnunet_constants.h"
30 #include "gnunet_datastore_service.h"
31 #include "gnunet_dht_service.h"
32 #include "gnunet_fs_service.h"
33 #include "gnunet_block_lib.h"
38 * Size of the individual blocks used for file-sharing.
40 #define DBLOCK_SIZE (32*1024)
43 * Pick a multiple of 2 here to achive 8-byte alignment! We also
44 * probably want DBlocks to have (roughly) the same size as IBlocks.
45 * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128
46 * byte = 2 * 512 bits). DO NOT CHANGE!
48 #define CHK_PER_INODE 256
51 * Maximum size for a file to be considered for inlining in a
54 #define MAX_INLINE_SIZE 65536
57 * Name of the directory with top-level searches.
59 #define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search"
62 * Name of the directory with sub-searches (namespace-updates).
64 #define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child"
67 * Name of the directory with master downloads (not associated
68 * with search or part of another download).
70 #define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download"
73 * Name of the directory with downloads that are part of another
74 * download or a search.
76 #define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child"
79 * Name of the directory with publishing operations.
81 #define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish"
84 * Name of the directory with files that are being published
86 #define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file"
89 * Name of the directory with unindex operations.
91 #define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex"
95 * @brief complete information needed
102 * Total size of the file in bytes. (network byte order (!))
104 uint64_t file_length;
107 * Query and key of the top GNUNET_EC_IBlock.
109 struct ContentHashKey chk;
115 * Information about a file and its location
116 * (peer claiming to share the file).
121 * Information about the shared file.
123 struct FileIdentifier fi;
126 * Identity of the peer sharing the file.
128 struct GNUNET_CRYPTO_EccPublicKey peer;
131 * Time when this location URI expires.
133 struct GNUNET_TIME_Absolute expirationTime;
136 * RSA signature over the GNUNET_EC_FileIdentifier,
137 * GNUNET_hash of the peer and expiration time.
139 struct GNUNET_CRYPTO_EccSignature contentSignature;
146 enum GNUNET_FS_UriType
149 * Content-hash-key (simple file).
154 * Signed key space (file in namespace).
159 * Keyword search key (query with keywords).
164 * Location (chk with identity of hosting peer).
171 * A Universal Resource Identifier (URI), opaque.
178 enum GNUNET_FS_UriType type;
185 * Keywords start with a '+' if they are mandatory (in which
186 * case the '+' is NOT part of the keyword) and with a simple
187 * space if they are optional (in which case the space is ALSO
188 * not part of the actual keyword).
190 * Double-quotes to protect spaces and %-encoding are NOT used
191 * internally (only in URI-strings).
196 * Size of the keywords array.
198 unsigned int keywordCount;
204 * Identifier of the namespace.
206 struct GNUNET_CRYPTO_EccPublicKey ns;
209 * Human-readable identifier chosen for this
210 * entry in the namespace.
216 * Information needed to retrieve a file (content-hash-key
219 struct FileIdentifier chk;
222 * Information needed to retrieve a file including signed
223 * location (identity of a peer) of the content.
232 * Information for a file or directory that is
233 * about to be published.
235 struct GNUNET_FS_FileInformation
239 * Files in a directory are kept as a linked list.
241 struct GNUNET_FS_FileInformation *next;
244 * If this is a file in a directory, "dir" refers to
245 * the directory; otherwise NULL.
247 struct GNUNET_FS_FileInformation *dir;
250 * Handle to the master context.
252 struct GNUNET_FS_Handle *h;
255 * Pointer kept for the client.
260 * Metadata to use for the file.
262 struct GNUNET_CONTAINER_MetaData *meta;
265 * Keywords to use for KBlocks.
267 struct GNUNET_FS_Uri *keywords;
270 * CHK for this file or directory. NULL if
271 * we have not yet computed it.
273 struct GNUNET_FS_Uri *chk_uri;
276 * Block options for the file.
278 struct GNUNET_FS_BlockOptions bo;
281 * At what time did we start this upload?
283 struct GNUNET_TIME_Absolute start_time;
286 * Under what filename is this struct serialized
287 * (for operational persistence). Should be determined
293 * Encoder being used to publish this file.
295 struct GNUNET_FS_TreeEncoder *te;
298 * Error message (non-NULL if this operation failed).
303 * Name of the file or directory (must be an absolute path).
308 * Data describing either the file or the directory.
320 * Function that can be used to read the data for the file.
322 GNUNET_FS_DataReader reader;
325 * Closure for reader.
330 * If this file is being indexed, this value is set to the hash
331 * over the entire file (when the indexing process is started).
332 * Otherwise this field is not used.
334 struct GNUNET_HashCode file_id;
337 * Size of the file (in bytes).
342 * Should the file be indexed or inserted?
347 * Is "file_id" already valid? Set to GNUNET_YES once the hash
348 * has been calculated.
353 * Has the service confirmed our INDEX_START request?
354 * GNUNET_YES if this step has been completed.
356 int index_start_confirmed;
361 * Data for a directory.
367 * Linked list of entries in the directory.
369 struct GNUNET_FS_FileInformation *entries;
372 * Size of the directory itself (in bytes); 0 if the
373 * size has not yet been calculated.
378 * Pointer to the data for the directory (or NULL if not
388 * Is this struct for a file or directory?
393 * Are we done publishing this file?
401 * The job is now ready to run and should use the given client
402 * handle to communicate with the FS service.
405 * @param client handle to use for FS communication
407 typedef void (*GNUNET_FS_QueueStart) (void *cls,
408 struct GNUNET_CLIENT_Connection * client);
412 * The job must now stop to run and should destry the client handle as
413 * soon as possible (ideally prior to returning).
415 typedef void (*GNUNET_FS_QueueStop) (void *cls);
420 * Priorities for the queue.
422 enum GNUNET_FS_QueuePriority
425 * This is a probe (low priority).
427 GNUNET_FS_QUEUE_PRIORITY_PROBE,
432 GNUNET_FS_QUEUE_PRIORITY_NORMAL
437 * Entry in the job queue.
439 struct GNUNET_FS_QueueEntry
442 * This is a linked list.
444 struct GNUNET_FS_QueueEntry *next;
447 * This is a linked list.
449 struct GNUNET_FS_QueueEntry *prev;
452 * Function to call when the job is started.
454 GNUNET_FS_QueueStart start;
457 * Function to call when the job needs to stop (or is done / dequeued).
459 GNUNET_FS_QueueStop stop;
462 * Closure for start and stop.
467 * Handle to FS primary context.
469 struct GNUNET_FS_Handle *h;
472 * Client handle, or NULL if job is not running.
474 struct GNUNET_CLIENT_Connection *client;
477 * Time the job was originally queued.
479 struct GNUNET_TIME_Absolute queue_time;
482 * Time the job was started last.
484 struct GNUNET_TIME_Absolute start_time;
487 * Total amount of time the job has been running (except for the
490 struct GNUNET_TIME_Relative run_time;
493 * How many blocks do the active downloads have?
498 * How important is this download?
500 enum GNUNET_FS_QueuePriority priority;
503 * How often have we (re)started this download?
505 unsigned int start_times;
511 * Information we store for each search result.
513 struct GNUNET_FS_SearchResult
517 * File-sharing context this result belongs to.
519 struct GNUNET_FS_Handle *h;
522 * Search context this result belongs to; can be NULL
523 * for probes that come from a directory result.
525 struct GNUNET_FS_SearchContext *sc;
528 * URI to which this search result refers to.
530 struct GNUNET_FS_Uri *uri;
533 * Metadata for the search result.
535 struct GNUNET_CONTAINER_MetaData *meta;
538 * Client info for this search result.
543 * ID of a job that is currently probing this results' availability
544 * (NULL if we are not currently probing).
546 struct GNUNET_FS_DownloadContext *probe_ctx;
549 * ID of an associated download based on this search result (or
552 struct GNUNET_FS_DownloadContext *download;
555 * If this search result triggered an update search, this field
556 * links to the update search.
558 struct GNUNET_FS_SearchContext *update_search;
561 * Name under which this search result is stored on disk.
566 * Bitmap that specifies precisely which keywords have been matched already.
568 uint8_t *keyword_bitmap;
571 * Key for the search result
573 struct GNUNET_HashCode key;
576 * ID of the task that will clean up the probe_ctx should it not
577 * complete on time (and that will need to be cancelled if we clean
578 * up the search result before then).
580 GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task;
583 * Task we use to report periodically to the application that the
584 * probe is still running.
586 GNUNET_SCHEDULER_TaskIdentifier probe_ping_task;
589 * When did the current probe become active?
591 struct GNUNET_TIME_Absolute probe_active_time;
594 * How much longer should we run the current probe before giving up?
596 struct GNUNET_TIME_Relative remaining_probe_time;
599 * Anonymity level to use for probes using this search result.
604 * Number of mandatory keywords for which we have NOT yet found the
605 * search result; when this value hits zero, the search result is
606 * given to the callback.
608 uint32_t mandatory_missing;
611 * Number of optional keywords under which this result was also
614 uint32_t optional_support;
617 * Number of availability tests that have succeeded for this result.
619 uint32_t availability_success;
622 * Number of availability trials that we have performed for this
625 uint32_t availability_trials;
631 * Add a job to the queue.
633 * @param h handle to the overall FS state
634 * @param start function to call to begin the job
635 * @param stop function to call to pause the job, or on dequeue (if the job was running)
636 * @param cls closure for start and stop
637 * @param blocks number of blocks this download has
638 * @param priority how important is this download
639 * @return queue handle
641 struct GNUNET_FS_QueueEntry *
642 GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start,
643 GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks,
644 enum GNUNET_FS_QueuePriority priority);
648 * Dequeue a job from the queue.
650 * @param qe handle for the job
653 GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qe);
657 * Function that provides data by reading from a file.
659 * @param cls closure (points to the file information)
660 * @param offset offset to read from; it is possible
661 * that the caller might need to go backwards
663 * @param max maximum number of bytes that should be
664 * copied to buf; readers are not allowed
665 * to provide less data unless there is an error;
666 * a value of "0" will be used at the end to allow
667 * the reader to clean up its internal state
668 * @param buf where the reader should write the data
669 * @param emsg location for the reader to store an error message
670 * @return number of bytes written, usually "max", 0 on error
673 GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf,
678 * Create the closure for the 'GNUNET_FS_data_reader_file_' callback.
680 * @param filename file to read
681 * @return closure to use
684 GNUNET_FS_make_file_reader_context_ (const char *filename);
689 * Function that provides data by copying from a buffer.
691 * @param cls closure (points to the buffer)
692 * @param offset offset to read from; it is possible
693 * that the caller might need to go backwards
695 * @param max maximum number of bytes that should be
696 * copied to buf; readers are not allowed
697 * to provide less data unless there is an error;
698 * a value of "0" will be used at the end to allow
699 * the reader to clean up its internal state
700 * @param buf where the reader should write the data
701 * @param emsg location for the reader to store an error message
702 * @return number of bytes written, usually "max", 0 on error
705 GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf,
709 * Notification of FS that a search probe has made progress.
710 * This function is used INSTEAD of the client's event handler
711 * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set.
713 * @param cls closure, always NULL (!), actual closure
714 * is in the client-context of the info struct
715 * @param info details about the event, specifying the event type
716 * and various bits about the event
717 * @return client-context (for the next progress call
718 * for this operation; should be set to NULL for
719 * SUSPEND and STOPPED events). The value returned
720 * will be passed to future callbacks in the respective
721 * field in the GNUNET_FS_ProgressInfo struct.
724 GNUNET_FS_search_probe_progress_ (void *cls,
725 const struct GNUNET_FS_ProgressInfo *info);
729 * Main function that performs the upload.
731 * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
732 * @param tc task context
735 GNUNET_FS_publish_main_ (void *cls,
736 const struct GNUNET_SCHEDULER_TaskContext *tc);
740 * Function called once the hash of the file
741 * that is being unindexed has been computed.
743 * @param cls closure, unindex context
744 * @param file_id computed hash, NULL on error
747 GNUNET_FS_unindex_process_hash_ (void *cls, const struct GNUNET_HashCode * file_id);
751 * Extract the keywords for KBlock removal
753 * @param uc context for the unindex operation.
756 GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc);
760 * If necessary, connect to the datastore and remove the KBlocks.
762 * @param uc context for the unindex operation.
765 GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc);
769 * Fill in all of the generic fields for a publish event and call the
772 * @param pi structure to fill in
773 * @param pc overall publishing context
774 * @param p file information for the file being published
775 * @param offset where in the file are we so far
776 * @return value returned from callback
779 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
780 struct GNUNET_FS_PublishContext *pc,
781 const struct GNUNET_FS_FileInformation *p,
786 * Fill in all of the generic fields for a download event and call the
789 * @param pi structure to fill in
790 * @param dc overall download context
793 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
794 struct GNUNET_FS_DownloadContext *dc);
798 * Task that creates the initial (top-level) download
799 * request for the file.
801 * @param cls the 'struct GNUNET_FS_DownloadContext'
802 * @param tc scheduler context
805 GNUNET_FS_download_start_task_ (void *cls,
806 const struct GNUNET_SCHEDULER_TaskContext *tc);
811 * Fill in all of the generic fields for
812 * an unindex event and call the callback.
814 * @param pi structure to fill in
815 * @param uc overall unindex context
816 * @param offset where we are in the file (for progress)
819 GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
820 struct GNUNET_FS_UnindexContext *uc,
824 * Fill in all of the generic fields for a search event and
827 * @param pi structure to fill in
828 * @param h file-sharing handle
829 * @param sc overall search context
830 * @return value returned by the callback
833 GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
834 struct GNUNET_FS_Handle *h,
835 struct GNUNET_FS_SearchContext *sc);
839 * Connect to the datastore and remove the blocks.
841 * @param uc context for the unindex operation.
844 GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc);
847 * Build the request and actually initiate the search using the
850 * @param sc search context
851 * @return GNUNET_OK on success, GNUNET_SYSERR on error
854 GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc);
857 * Start the downloading process (by entering the queue).
859 * @param dc our download context
862 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc);
866 * Start download probes for the given search result.
868 * @param sr the search result
871 GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr);
874 * Remove serialization/deserialization file from disk.
876 * @param h master context
877 * @param ext component of the path
878 * @param ent entity identifier
881 GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext,
886 * Remove serialization/deserialization directory from disk.
888 * @param h master context
889 * @param ext component of the path
890 * @param uni unique name of parent
893 GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext,
898 * Synchronize this file-information struct with its mirror
899 * on disk. Note that all internal FS-operations that change
900 * file information data should already call "sync" internally,
901 * so this function is likely not useful for clients.
903 * @param fi the struct to sync
906 GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f);
909 * Synchronize this publishing struct with its mirror
910 * on disk. Note that all internal FS-operations that change
911 * publishing structs should already call "sync" internally,
912 * so this function is likely not useful for clients.
914 * @param pc the struct to sync
917 GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc);
920 * Synchronize this unindex struct with its mirror
921 * on disk. Note that all internal FS-operations that change
922 * publishing structs should already call "sync" internally,
923 * so this function is likely not useful for clients.
925 * @param uc the struct to sync
928 GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc);
931 * Synchronize this search struct with its mirror
932 * on disk. Note that all internal FS-operations that change
933 * publishing structs should already call "sync" internally,
934 * so this function is likely not useful for clients.
936 * @param sc the struct to sync
939 GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc);
942 * Synchronize this search result with its mirror
943 * on disk. Note that all internal FS-operations that change
944 * publishing structs should already call "sync" internally,
945 * so this function is likely not useful for clients.
947 * @param sr the struct to sync
950 GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr);
953 * Synchronize this download struct with its mirror
954 * on disk. Note that all internal FS-operations that change
955 * publishing structs should already call "sync" internally,
956 * so this function is likely not useful for clients.
958 * @param dc the struct to sync
961 GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc);
964 * Create SUSPEND event for the given publish operation
965 * and then clean up our state (without stop signal).
967 * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
970 GNUNET_FS_publish_signal_suspend_ (void *cls);
973 * Create SUSPEND event for the given search operation
974 * and then clean up our state (without stop signal).
976 * @param cls the 'struct GNUNET_FS_SearchContext' to signal for
979 GNUNET_FS_search_signal_suspend_ (void *cls);
982 * Create SUSPEND event for the given download operation
983 * and then clean up our state (without stop signal).
985 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
988 GNUNET_FS_download_signal_suspend_ (void *cls);
991 * Create SUSPEND event for the given unindex operation
992 * and then clean up our state (without stop signal).
994 * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for
997 GNUNET_FS_unindex_signal_suspend_ (void *cls);
1000 * Function signature of the functions that can be called
1001 * to trigger suspend signals and clean-up for top-level
1004 * @param cls closure
1006 typedef void (*SuspendSignalFunction) (void *cls);
1009 * We track all of the top-level activities of FS
1010 * so that we can signal 'suspend' on shutdown.
1012 struct TopLevelActivity
1015 * This is a doubly-linked list.
1017 struct TopLevelActivity *next;
1020 * This is a doubly-linked list.
1022 struct TopLevelActivity *prev;
1025 * Function to call for suspend-signalling and clean up.
1027 SuspendSignalFunction ssf;
1030 * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*)
1037 * Create a top-level activity entry.
1039 * @param h global fs handle
1040 * @param ssf suspend signal function to use
1041 * @param ssf_cls closure for ssf
1042 * @return fresh top-level activity handle
1044 struct TopLevelActivity *
1045 GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf,
1050 * Destroy a top-level activity entry.
1052 * @param h global fs handle
1053 * @param top top level activity entry
1056 GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top);
1061 * Master context for most FS operations.
1063 struct GNUNET_FS_Handle
1066 * Configuration to use.
1068 const struct GNUNET_CONFIGURATION_Handle *cfg;
1071 * Name of our client.
1076 * Function to call with updates on our progress.
1078 GNUNET_FS_ProgressCallback upcb;
1086 * Head of DLL of top-level activities.
1088 struct TopLevelActivity *top_head;
1091 * Tail of DLL of top-level activities.
1093 struct TopLevelActivity *top_tail;
1096 * Head of DLL of running jobs.
1098 struct GNUNET_FS_QueueEntry *running_head;
1101 * Tail of DLL of running jobs.
1103 struct GNUNET_FS_QueueEntry *running_tail;
1106 * Head of DLL of pending jobs.
1108 struct GNUNET_FS_QueueEntry *pending_head;
1111 * Tail of DLL of pending jobs.
1113 struct GNUNET_FS_QueueEntry *pending_tail;
1116 * Task that processes the jobs in the running and pending queues
1117 * (and moves jobs around as needed).
1119 GNUNET_SCHEDULER_TaskIdentifier queue_job;
1122 * Average time we take for a single request to be satisfied.
1123 * FIXME: not yet calcualted properly...
1125 struct GNUNET_TIME_Relative avg_block_latency;
1128 * How many actual downloads do we have running right now?
1130 unsigned int active_downloads;
1133 * How many blocks do the active downloads have?
1135 unsigned int active_blocks;
1140 enum GNUNET_FS_Flags flags;
1143 * Maximum number of parallel downloads.
1145 unsigned int max_parallel_downloads;
1148 * Maximum number of parallel requests.
1150 unsigned int max_parallel_requests;
1156 * Handle for controlling a publication process.
1158 struct GNUNET_FS_PublishContext
1161 * Handle to the global fs context.
1163 struct GNUNET_FS_Handle *h;
1166 * Our top-level activity entry (if we are top-level, otherwise NULL).
1168 struct TopLevelActivity *top;
1171 * File-structure that is being shared.
1173 struct GNUNET_FS_FileInformation *fi;
1176 * Namespace that we are publishing in, NULL if we have no namespace.
1178 struct GNUNET_CRYPTO_EccPrivateKey *ns;
1181 * ID of the content in the namespace, NULL if we have no namespace.
1186 * ID for future updates, NULL if we have no namespace or no updates.
1191 * Filename used for serializing information about this operation
1192 * (should be determined using 'mktemp').
1194 char *serialization;
1197 * Our own client handle for the FS service; only briefly used when
1198 * we start to index a file, otherwise NULL.
1200 struct GNUNET_CLIENT_Connection *client;
1203 * Current position in the file-tree for the upload.
1205 struct GNUNET_FS_FileInformation *fi_pos;
1208 * Non-null if we are currently hashing a file.
1210 struct GNUNET_CRYPTO_FileHashContext *fhc;
1213 * Connection to the datastore service.
1215 struct GNUNET_DATASTORE_Handle *dsh;
1218 * Queue entry for reservation/unreservation.
1220 struct GNUNET_DATASTORE_QueueEntry *qre;
1223 * Context for SKS publishing operation that is part of this publishing operation
1224 * (NULL if not active).
1226 struct GNUNET_FS_PublishSksContext *sks_pc;
1229 * Context for KSK publishing operation that is part of this publishing operation
1230 * (NULL if not active).
1232 struct GNUNET_FS_PublishKskContext *ksk_pc;
1235 * ID of the task performing the upload. NO_TASK if the upload has
1238 GNUNET_SCHEDULER_TaskIdentifier upload_task;
1241 * Storage space to reserve for the operation.
1243 uint64_t reserve_space;
1246 * Overall number of entries to reserve for the
1247 * publish operation.
1249 uint32_t reserve_entries;
1252 * Options for publishing.
1254 enum GNUNET_FS_PublishOptions options;
1257 * Space reservation ID with datastore service
1263 * Set to GNUNET_YES if all processing has completed.
1268 * Flag set to GNUNET_YES if the next callback from
1269 * GNUNET_FS_file_information_inspect should be skipped because it
1270 * is for the directory which was already processed with the parent.
1272 int skip_next_fi_callback;
1277 * Phases of unindex processing (state machine).
1282 * We're currently hashing the file.
1284 UNINDEX_STATE_HASHING = 0,
1287 * We're telling the datastore to delete
1288 * the respective DBlocks and IBlocks.
1290 UNINDEX_STATE_DS_REMOVE = 1,
1293 * Find out which keywords apply.
1295 UNINDEX_STATE_EXTRACT_KEYWORDS = 2,
1298 * We're telling the datastore to remove KBlocks.
1300 UNINDEX_STATE_DS_REMOVE_KBLOCKS = 3,
1303 * We're notifying the FS service about
1306 UNINDEX_STATE_FS_NOTIFY = 4,
1311 UNINDEX_STATE_COMPLETE = 5,
1314 * We've encountered a fatal error.
1316 UNINDEX_STATE_ERROR = 6
1321 * Handle for controlling an unindexing operation.
1323 struct GNUNET_FS_UnindexContext
1327 * The content hash key of the last block we processed, will in the
1328 * end be set to the CHK from the URI. Used to remove the KBlocks.
1330 struct ContentHashKey chk;
1333 * Global FS context.
1335 struct GNUNET_FS_Handle *h;
1338 * Our top-level activity entry.
1340 struct TopLevelActivity *top;
1343 * Directory scanner to find keywords (KBlock removal).
1345 struct GNUNET_FS_DirScanner *dscan;
1348 * Keywords found (telling us which KBlocks to remove).
1350 struct GNUNET_FS_Uri *ksk_uri;
1353 * Current offset in KSK removal.
1355 uint32_t ksk_offset;
1358 * Name of the file that we are unindexing.
1363 * Short name under which we are serializing the state of this operation.
1365 char *serialization;
1368 * Connection to the FS service, only valid during the
1369 * UNINDEX_STATE_FS_NOTIFY phase.
1371 struct GNUNET_CLIENT_Connection *client;
1374 * Connection to the datastore service, only valid during the
1375 * UNINDEX_STATE_DS_NOTIFY phase.
1377 struct GNUNET_DATASTORE_Handle *dsh;
1380 * Pointer kept for the client.
1385 * Merkle-ish tree encoder context.
1387 struct GNUNET_FS_TreeEncoder *tc;
1390 * Handle used to read the file.
1392 struct GNUNET_DISK_FileHandle *fh;
1395 * Handle to datastore 'get_key' operation issued for
1396 * obtaining KBlocks.
1398 struct GNUNET_DATASTORE_QueueEntry *dqe;
1401 * Current key for decrypting UBLocks from 'get_key' operation.
1403 struct GNUNET_HashCode ukey;
1406 * Current query of 'get_key' operation.
1408 struct GNUNET_HashCode uquery;
1411 * First content UID, 0 for none.
1416 * Error message, NULL on success.
1421 * Context for hashing of the file.
1423 struct GNUNET_CRYPTO_FileHashContext *fhc;
1426 * Overall size of the file.
1431 * Random offset given to 'GNUNET_DATASTORE_get_key'.
1436 * When did we start?
1438 struct GNUNET_TIME_Absolute start_time;
1441 * Hash of the file's contents (once computed).
1443 struct GNUNET_HashCode file_id;
1446 * Current operatinonal phase.
1448 enum UnindexState state;
1454 * Information we keep for each keyword in
1457 struct SearchRequestEntry
1461 * Hash of the public key, also known as the query.
1463 struct GNUNET_HashCode uquery;
1466 * Derived public key, hashes to 'uquery'.
1468 struct GNUNET_CRYPTO_EccPublicKey dpub;
1471 * The original keyword, used to derive the
1472 * key (for decrypting the UBlock).
1477 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1478 * was found under this keyword. Note that the entries will point
1479 * to the same locations as those in the master result map (in
1480 * "struct GNUNET_FS_SearchContext"), so they should not be freed.
1481 * The key for each entry is the XOR of the key and query in the CHK
1482 * URI (as a unique identifier for the search result).
1484 struct GNUNET_CONTAINER_MultiHashMap *results;
1487 * Is this keyword a mandatory keyword
1488 * (started with '+')?
1496 * Handle for controlling a search.
1498 struct GNUNET_FS_SearchContext
1501 * Handle to the global FS context.
1503 struct GNUNET_FS_Handle *h;
1506 * Our top-level activity entry (if we are top-level, otherwise NULL).
1508 struct TopLevelActivity *top;
1511 * List of keywords that we're looking for.
1513 struct GNUNET_FS_Uri *uri;
1516 * For update-searches, link to the search result that triggered
1517 * the update search; otherwise NULL.
1519 struct GNUNET_FS_SearchResult *psearch_result;
1522 * Connection to the FS service.
1524 struct GNUNET_CLIENT_Connection *client;
1527 * Pointer we keep for the client.
1532 * Name of the file on disk we use for persistence.
1534 char *serialization;
1537 * Error message (non-NULL if this operation failed).
1542 * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
1543 * was found in the search. The key for each entry is the XOR of
1544 * the key and query in the CHK URI (as a unique identifier for the
1547 struct GNUNET_CONTAINER_MultiHashMap *master_result_map;
1550 * Per-keyword information for a keyword search. This array will
1551 * have exactly as many entries as there were keywords.
1553 struct SearchRequestEntry *requests;
1556 * When did we start?
1558 struct GNUNET_TIME_Absolute start_time;
1561 * How long to wait before we try to reconnect to FS service?
1563 struct GNUNET_TIME_Relative reconnect_backoff;
1566 * ID of a task that is using this struct and that must be cancelled
1567 * when the search is being stopped (if not
1568 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1569 * artificial delay when trying to reconnect to the FS service.
1571 GNUNET_SCHEDULER_TaskIdentifier task;
1574 * How many of the entries in the search request
1575 * map have been passed to the service so far?
1577 unsigned int search_request_map_offset;
1580 * How many of the keywords in the KSK
1581 * map have been passed to the service so far?
1583 unsigned int keyword_offset;
1586 * Anonymity level for the search.
1591 * Number of mandatory keywords in this query.
1593 uint32_t mandatory_count;
1596 * Options for the search.
1598 enum GNUNET_FS_SearchOptions options;
1603 * FSM for possible states a block can go through. The typical
1604 * order of progression is linear through the states, alternatives
1605 * are documented in the comments.
1607 enum BlockRequestState
1610 * Initial state, block has only been allocated (since it is
1611 * relevant to the overall download request).
1616 * We've checked the block on the path down the tree, and the
1617 * content on disk did match the desired CHK, but not all
1618 * the way down, so at the bottom some blocks will still
1619 * need to be reconstructed).
1621 BRS_RECONSTRUCT_DOWN = 1,
1624 * We've calculated the CHK bottom-up based on the meta data.
1625 * This may work, but if it did we have to write the meta data to
1626 * disk at the end (and we still need to check against the
1629 BRS_RECONSTRUCT_META_UP = 2,
1632 * We've calculated the CHK bottom-up based on what we have on
1633 * disk, which may not be what the desired CHK is. If the
1634 * reconstructed CHKs match whatever comes from above, we're
1635 * done with the respective subtree.
1637 BRS_RECONSTRUCT_UP = 3,
1640 * We've determined the real, desired CHK for this block
1641 * (full tree reconstruction failed), request is now pending.
1642 * If the CHK that bubbled up through reconstruction did match
1643 * the top-level request, the state machine for the subtree
1644 * would have moved to BRS_DOWNLOAD_UP.
1649 * We've successfully downloaded this block, but the children
1650 * still need to be either downloaded or verified (download
1651 * request propagates down). If the download fails, the
1652 * state machine for this block may move to
1653 * BRS_DOWNLOAD_ERROR instead.
1655 BRS_DOWNLOAD_DOWN = 5,
1658 * This block and all of its children have been downloaded
1659 * successfully (full completion propagates up).
1661 BRS_DOWNLOAD_UP = 6,
1664 * We got a block back that matched the query but did not hash to
1665 * the key (malicious publisher or hash collision); this block
1666 * can never be downloaded (error propagates up).
1673 * Information about an active download request.
1675 struct DownloadRequest
1678 * While pending, we keep all download requests in a doubly-linked list.
1680 struct DownloadRequest *next;
1683 * While pending, we keep all download requests in a doubly-linked list.
1685 struct DownloadRequest *prev;
1688 * Parent in the CHK-tree.
1690 struct DownloadRequest *parent;
1693 * Array (!) of child-requests, or NULL for the bottom of the tree.
1695 struct DownloadRequest **children;
1698 * CHK for the request for this block (set during reconstruction
1699 * to what we have on disk, later to what we want to have).
1701 struct ContentHashKey chk;
1704 * Offset of the corresponding block. Specifically, first (!) byte of
1705 * the first DBLOCK in the subtree induced by block represented by
1711 * Number of entries in 'children' array.
1713 unsigned int num_children;
1716 * Depth of the corresponding block in the tree. 0==DBLOCKs.
1721 * Offset of the CHK for this block in the parent block
1723 unsigned int chk_idx;
1728 enum BlockRequestState state;
1731 * GNUNET_YES if this entry is in the pending list.
1739 * (recursively) free download request structure
1741 * @param dr request to free
1744 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr);
1748 * Context for controlling a download.
1750 struct GNUNET_FS_DownloadContext
1754 * Global FS context.
1756 struct GNUNET_FS_Handle *h;
1759 * Our top-level activity entry (if we are top-level, otherwise NULL).
1761 struct TopLevelActivity *top;
1764 * Connection to the FS service.
1766 struct GNUNET_CLIENT_Connection *client;
1769 * Parent download (used when downloading files
1772 struct GNUNET_FS_DownloadContext *parent;
1775 * Associated search (used when downloading files
1776 * based on search results), or NULL for none.
1778 struct GNUNET_FS_SearchResult *search;
1781 * Head of list of child downloads.
1783 struct GNUNET_FS_DownloadContext *child_head;
1786 * Tail of list of child downloads.
1788 struct GNUNET_FS_DownloadContext *child_tail;
1791 * Previous download belonging to the same parent.
1793 struct GNUNET_FS_DownloadContext *prev;
1796 * Next download belonging to the same parent.
1798 struct GNUNET_FS_DownloadContext *next;
1801 * Context kept for the client.
1806 * URI that identifies the file that we are downloading.
1808 struct GNUNET_FS_Uri *uri;
1811 * Known meta-data for the file (can be NULL).
1813 struct GNUNET_CONTAINER_MetaData *meta;
1816 * Error message, NULL if we're doing OK.
1821 * Random portion of filename we use for syncing state of this
1824 char *serialization;
1827 * Where are we writing the data (name of the
1828 * file, can be NULL!).
1833 * Where are we writing the data temporarily (name of the
1834 * file, can be NULL!); used if we do not have a permanent
1835 * name and we are a directory and we do a recursive download.
1837 char *temp_filename;
1840 * Our entry in the job queue.
1842 struct GNUNET_FS_QueueEntry *job_queue;
1845 * Non-NULL if we are currently having a request for
1846 * transmission pending with the client handle.
1848 struct GNUNET_CLIENT_TransmitHandle *th;
1851 * Tree encoder used for the reconstruction.
1853 struct GNUNET_FS_TreeEncoder *te;
1856 * File handle for reading data from an existing file
1857 * (to pass to tree encoder).
1859 struct GNUNET_DISK_FileHandle *rfh;
1862 * Map of active requests (those waiting for a response). The key
1863 * is the hash of the encryped block (aka query).
1865 struct GNUNET_CONTAINER_MultiHashMap *active;
1868 * Head of linked list of pending requests.
1870 struct DownloadRequest *pending_head;
1873 * Head of linked list of pending requests.
1875 struct DownloadRequest *pending_tail;
1878 * Top-level download request.
1880 struct DownloadRequest *top_request;
1883 * Identity of the peer having the content, or all-zeros
1884 * if we don't know of such a peer.
1886 struct GNUNET_PeerIdentity target;
1889 * ID of a task that is using this struct and that must be cancelled
1890 * when the download is being stopped (if not
1891 * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some
1892 * artificial delay when trying to reconnect to the FS service or
1893 * the task processing incrementally the data on disk, or the
1894 * task requesting blocks, etc.
1896 GNUNET_SCHEDULER_TaskIdentifier task;
1899 * What is the first offset that we're interested
1905 * How many bytes starting from offset are desired?
1906 * This is NOT the overall length of the file!
1911 * How many bytes have we already received within
1912 * the specified range (DBlocks only).
1917 * What was the size of the file on disk that we're downloading
1918 * before we started? Used to detect if there is a point in
1919 * checking an existing block on disk for matching the desired
1920 * content. 0 if the file did not exist already.
1922 uint64_t old_file_size;
1925 * Time download was started.
1927 struct GNUNET_TIME_Absolute start_time;
1930 * How long to wait before we try to reconnect to FS service?
1932 struct GNUNET_TIME_Relative reconnect_backoff;
1935 * Desired level of anonymity.
1940 * The depth of the file-tree.
1942 unsigned int treedepth;
1945 * Options for the download.
1947 enum GNUNET_FS_DownloadOptions options;
1950 * Flag set upon transitive completion (includes child downloads).
1951 * This flag is only set to GNUNET_YES for directories where all
1952 * child-downloads have also completed (and signalled completion).
1957 * Have we started the receive continuation yet?
1962 * Are we ready to issue requests (reconstructions are finished)?
1971 /* end of fs_api.h */