From 0443654040fb277df95aae28a7b2a54a4f0a73bf Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 15 Nov 2011 09:38:48 +0000 Subject: [PATCH] dead code elimination, splitting fs.h into fs.h and fs_api.h --- src/fs/Makefile.am | 2 +- src/fs/fs.h | 2001 +------------------------------ src/fs/{fs.c => fs_api.c} | 4 +- src/fs/fs_api.h | 1907 +++++++++++++++++++++++++++++ src/fs/fs_directory.c | 2 +- src/fs/fs_download.c | 2 +- src/fs/fs_file_information.c | 2 +- src/fs/fs_getopt.c | 2 +- src/fs/fs_list_indexed.c | 2 +- src/fs/fs_misc.c | 2 +- src/fs/fs_namespace.c | 14 +- src/fs/fs_publish.c | 7 +- src/fs/fs_search.c | 7 +- src/fs/fs_test_lib.c | 2 +- src/fs/fs_tree.h | 2 +- src/fs/fs_unindex.c | 2 +- src/fs/fs_uri.c | 2 +- src/fs/gnunet-service-fs.c | 2 - src/fs/gnunet-service-fs.h | 23 +- src/fs/gnunet-service-fs_cp.c | 8 + src/fs/gnunet-service-fs_cp.h | 30 + src/fs/gnunet-service-fs_pr.c | 15 + src/fs/gnunet-service-fs_push.c | 11 + src/fs/test_fs_directory.c | 2 +- src/fs/test_fs_uri.c | 2 +- 25 files changed, 2019 insertions(+), 2036 deletions(-) rename src/fs/{fs.c => fs_api.c} (99%) create mode 100644 src/fs/fs_api.h diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 8cc0f9bf3..b1f15afa3 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -20,7 +20,7 @@ lib_LTLIBRARIES = libgnunetfs.la noinst_LIBRARIES = libgnunetfstest.a libgnunetfs_la_SOURCES = \ - fs.c fs.h \ + fs_api.c fs_api.h fs.h \ fs_directory.c \ fs_download.c \ fs_file_information.c \ diff --git a/src/fs/fs.h b/src/fs/fs.h index b5fdb1cc7..89a375022 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -34,202 +34,18 @@ #include "block_fs.h" -/** - * Maximum number of outgoing messages we queue per peer. - * - * Performance measurements for 2 peer setup for 50 MB file - * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): - * - * 2: 1700 kb/s, 1372 kb/s - * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s - * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s - * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s - * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s - * - * Conclusion: 16 seems to be a pretty good value (stable - * and high performance, no excessive memory use). - */ -#define MAX_QUEUE_PER_PEER 16 - -/** - * Maximum size of the datastore queue for P2P operations. - * Needs to be large enough to queue MAX_QUEUE_PER_PEER - * operations for roughly the number of active (connected) - * peers. - */ -#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) - -/** - * Maximum number of blocks we keep in memory for migration. - */ -#define MAX_MIGRATION_QUEUE 8 - -/** - * Blocks are at most migrated to this number of peers - * plus one, each time they are fetched from the database. - */ -#define MIGRATION_LIST_SIZE 2 - -/** - * To how many peers do we forward each migration block ultimately? - * This number must be smaller or equal to MIGRATION_LIST_SIZE. Using - * a smaller value allows for variation in available bandwidth (for - * migration) between the peers. - */ -#define MIGRATION_TARGET_COUNT 2 - -/** - * Ratio for moving average delay calculation. The previous - * average goes in with a factor of (n-1) into the calculation. - * Must be > 0. - */ -#define RUNAVG_DELAY_N 16 - /** * Size of the individual blocks used for file-sharing. */ #define DBLOCK_SIZE (32*1024) /** - * Maximum legal size for a kblock. - */ -#define MAX_KBLOCK_SIZE (60 * 1024) - -/** - * Maximum legal size for an sblock. - */ -#define MAX_SBLOCK_SIZE (60 * 1024) - -/** - * Maximum legal size for an nblock. - */ -#define MAX_NBLOCK_SIZE (60 * 1024) - -/** - * Pick a multiple of 2 here to achive 8-byte alignment! - * We also probably want DBlocks to have (roughly) the - * same size as IBlocks. With SHA-512, the optimal - * value is 32768 byte / 128 byte = 256 - * (128 byte = 2 * 512 bits). DO NOT CHANGE! - */ -#define CHK_PER_INODE 256 - - -/** - * Maximum size for a file to be considered for - * inlining in a directory. - */ -#define MAX_INLINE_SIZE 65536 - - -/** - * Blocksize to use when hashing files - * for indexing (blocksize for IO, not for - * the DBlocks). Larger blocksizes can - * be more efficient but will be more disruptive - * as far as the scheduler is concerned. + * Blocksize to use when hashing files for indexing (blocksize for IO, + * not for the DBlocks). Larger blocksizes can be more efficient but + * will be more disruptive as far as the scheduler is concerned. */ #define HASHING_BLOCKSIZE (1024 * 128) -/** - * Number of availability trials we perform per search result. - */ -#define AVAILABILITY_TRIALS_MAX 8 - -/** - * Length of the P2P success tracker. Note that - * having a very long list can also hurt performance. - */ -#define P2P_SUCCESS_LIST_SIZE 8 - - -/** - * Length of the CS-2-P success tracker. Note that - * having a very long list can also hurt performance. - */ -#define CS2P_SUCCESS_LIST_SIZE 8 - -/** - * How long are we willing to wait for the datastore to be ready to - * process a request for a query without priority? - */ -#define BASIC_DATASTORE_REQUEST_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) - - -/** - * How long are we willing to wait for the core to be ready to - * transmit a reply to the target peer (if we can not transmit - * until then, we will discard the reply). - */ -#define ACCEPTABLE_REPLY_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) - - -/** - * Bandwidth value of an (effectively) 0-priority query. - */ -#define QUERY_BANDWIDTH_VALUE 1 - -/** - * Bandwidth value of a 0-priority content (must be - * fairly high compared to query since content is - * typically significantly larger -- and more valueable - * since it can take many queries to get one piece of - * content). - */ -#define CONTENT_BANDWIDTH_VALUE 800 - -/** - * By which amount do we decrement the TTL for simple forwarding / - * indirection of the query; in milli-seconds. Set somewhat in - * accordance to your network latency (above the time it'll take you - * to send a packet and get a reply). - */ -#define TTL_DECREMENT 5000 - -/** - * Until which load do we consider the peer idle and do not - * charge at all? (should be larger than GNUNET_IDLE_LOAD_THRESHOLD used - * by the rest of the code)! - */ -#define IDLE_LOAD_THRESHOLD ((100 + GNUNET_CONSTANTS_IDLE_LOAD_THRESHOLD) / 2) - -/** - * Name of the directory with top-level searches. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" - -/** - * Name of the directory with sub-searches (namespace-updates). - */ -#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" - -/** - * Name of the directory with master downloads (not associated - * with search or part of another download). - */ -#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" - -/** - * Name of the directory with downloads that are part of another - * download or a search. - */ -#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" - -/** - * Name of the directory with publishing operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" - -/** - * Name of the directory with files that are being published - */ -#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" - -/** - * Name of the directory with unindex operations. - */ -#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" - /** * @brief content hash key @@ -248,1817 +64,6 @@ struct ContentHashKey }; -/** - * @brief complete information needed - * to download a file. - */ -struct FileIdentifier -{ - - /** - * Total size of the file in bytes. (network byte order (!)) - */ - uint64_t file_length; - - /** - * Query and key of the top GNUNET_EC_IBlock. - */ - struct ContentHashKey chk; - -}; - - -/** - * Information about a file and its location - * (peer claiming to share the file). - */ -struct Location -{ - /** - * Information about the shared file. - */ - struct FileIdentifier fi; - - /** - * Identity of the peer sharing the file. - */ - struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; - - /** - * Time when this location URI expires. - */ - struct GNUNET_TIME_Absolute expirationTime; - - /** - * RSA signature over the GNUNET_EC_FileIdentifier, - * GNUNET_hash of the peer and expiration time. - */ - struct GNUNET_CRYPTO_RsaSignature contentSignature; - -}; - -/** - * Types of URIs. - */ -enum uri_types -{ - /** - * Content-hash-key (simple file). - */ - chk, - - /** - * Signed key space (file in namespace). - */ - sks, - - /** - * Keyword search key (query with keywords). - */ - ksk, - - /** - * Location (chk with identity of hosting peer). - */ - loc -}; - -/** - * A Universal Resource Identifier (URI), opaque. - */ -struct GNUNET_FS_Uri -{ - /** - * Type of the URI. - */ - enum uri_types type; - - union - { - struct - { - /** - * Keywords start with a '+' if they are - * mandatory (in which case the '+' is NOT - * part of the keyword) and with a - * simple space if they are optional - * (in which case the space is ALSO not - * part of the actual keyword). - * - * Double-quotes to protect spaces and - * %-encoding are NOT used internally - * (only in URI-strings). - */ - char **keywords; - - /** - * Size of the keywords array. - */ - unsigned int keywordCount; - } ksk; - - struct - { - /** - * Hash of the public key for the namespace. - */ - GNUNET_HashCode namespace; - - /** - * Human-readable identifier chosen for this - * entry in the namespace. - */ - char *identifier; - } sks; - - /** - * Information needed to retrieve a file (content-hash-key - * plus file size). - */ - struct FileIdentifier chk; - - /** - * Information needed to retrieve a file including signed - * location (identity of a peer) of the content. - */ - struct Location loc; - } data; - -}; - - -/** - * Information for a file or directory that is - * about to be published. - */ -struct GNUNET_FS_FileInformation -{ - - /** - * Files in a directory are kept as a linked list. - */ - struct GNUNET_FS_FileInformation *next; - - /** - * If this is a file in a directory, "dir" refers to - * the directory; otherwise NULL. - */ - struct GNUNET_FS_FileInformation *dir; - - /** - * Handle to the master context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Metadata to use for the file. - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Keywords to use for KBlocks. - */ - struct GNUNET_FS_Uri *keywords; - - /** - * CHK for this file or directory. NULL if - * we have not yet computed it. - */ - struct GNUNET_FS_Uri *chk_uri; - - /** - * Block options for the file. - */ - struct GNUNET_FS_BlockOptions bo; - - /** - * At what time did we start this upload? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Under what filename is this struct serialized - * (for operational persistence). Should be determined - * using 'mktemp'. - */ - char *serialization; - - /** - * Encoder being used to publish this file. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Name of the file or directory (must be an absolute path). - */ - char *filename; - - /** - * Data describing either the file or the directory. - */ - union - { - - /** - * Data for a file. - */ - struct - { - - /** - * Function that can be used to read the data for the file. - */ - GNUNET_FS_DataReader reader; - - /** - * Closure for reader. - */ - void *reader_cls; - - /** - * If this file is being indexed, this value is set to the hash - * over the entire file (when the indexing process is started). - * Otherwise this field is not used. - */ - GNUNET_HashCode file_id; - - /** - * Size of the file (in bytes). - */ - uint64_t file_size; - - /** - * Should the file be indexed or inserted? - */ - int do_index; - - /** - * Is "file_id" already valid? Set to GNUNET_YES once the hash - * has been calculated. - */ - int have_hash; - - /** - * Has the service confirmed our INDEX_START request? - * GNUNET_YES if this step has been completed. - */ - int index_start_confirmed; - - } file; - - /** - * Data for a directory. - */ - struct - { - - /** - * Linked list of entries in the directory. - */ - struct GNUNET_FS_FileInformation *entries; - - /** - * Size of the directory itself (in bytes); 0 if the - * size has not yet been calculated. - */ - size_t dir_size; - - /** - * Pointer to the data for the directory (or NULL if not - * available). - */ - void *dir_data; - - } dir; - - } data; - - /** - * Is this struct for a file or directory? - */ - int is_directory; - - /** - * Are we done publishing this file? - */ - int is_published; - -}; - - -/** - * The job is now ready to run and should use the given client - * handle to communicate with the FS service. - * - * @param cls closure - * @param client handle to use for FS communication - */ -typedef void (*GNUNET_FS_QueueStart) (void *cls, - struct GNUNET_CLIENT_Connection * client); - - -/** - * The job must now stop to run and should destry the client handle as - * soon as possible (ideally prior to returning). - */ -typedef void (*GNUNET_FS_QueueStop) (void *cls); - - -/** - * Entry in the job queue. - */ -struct GNUNET_FS_QueueEntry -{ - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *next; - - /** - * This is a linked list. - */ - struct GNUNET_FS_QueueEntry *prev; - - /** - * Function to call when the job is started. - */ - GNUNET_FS_QueueStart start; - - /** - * Function to call when the job needs to stop (or is done / dequeued). - */ - GNUNET_FS_QueueStop stop; - - /** - * Closure for start and stop. - */ - void *cls; - - /** - * Handle to FS primary context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Client handle, or NULL if job is not running. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Time the job was originally queued. - */ - struct GNUNET_TIME_Absolute queue_time; - - /** - * Time the job was started last. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Total amount of time the job has been running (except for the - * current run). - */ - struct GNUNET_TIME_Relative run_time; - - /** - * How many blocks do the active downloads have? - */ - unsigned int blocks; - - /** - * How often have we (re)started this download? - */ - unsigned int start_times; - -}; - - - - -/** - * Information we store for each search result. - */ -struct GNUNET_FS_SearchResult -{ - - /** - * Search context this result belongs to. - */ - struct GNUNET_FS_SearchContext *sc; - - /** - * URI to which this search result refers to. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Metadata for the search result. - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Client info for this search result. - */ - void *client_info; - - /** - * ID of a job that is currently probing this results' availability - * (NULL if we are not currently probing). - */ - struct GNUNET_FS_DownloadContext *probe_ctx; - - /** - * ID of an associated download based on this search result (or - * NULL for none). - */ - struct GNUNET_FS_DownloadContext *download; - - /** - * If this search result triggered an update search, this field - * links to the update search. - */ - struct GNUNET_FS_SearchContext *update_search; - - /** - * Name under which this search result is stored on disk. - */ - char *serialization; - - /** - * Key for the search result - */ - GNUNET_HashCode key; - - /** - * ID of the task that will clean up the probe_ctx should it not - * complete on time (and that will need to be cancelled if we clean - * up the search result before then). - */ - GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task; - - /** - * When did the current probe become active? - */ - struct GNUNET_TIME_Absolute probe_active_time; - - /** - * How much longer should we run the current probe before giving up? - */ - struct GNUNET_TIME_Relative remaining_probe_time; - - /** - * Number of mandatory keywords for which we have NOT yet found the - * search result; when this value hits zero, the search result is - * given to the callback. - */ - uint32_t mandatory_missing; - - /** - * Number of optional keywords under which this result was also - * found. - */ - uint32_t optional_support; - - /** - * Number of availability tests that have succeeded for this result. - */ - uint32_t availability_success; - - /** - * Number of availability trials that we have performed for this - * search result. - */ - uint32_t availability_trials; - -}; - - -/** - * Add a job to the queue. - * - * @param h handle to the overall FS state - * @param start function to call to begin the job - * @param stop function to call to pause the job, or on dequeue (if the job was running) - * @param cls closure for start and stop - * @param blocks number of blocks this download has - * @return queue handle - */ -struct GNUNET_FS_QueueEntry * -GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, - GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks); - - -/** - * Dequeue a job from the queue. - * @param qh handle for the job - */ -void -GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh); - - -/** - * Function that provides data by reading from a file. - * - * @param cls closure (points to the file information) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg); - - -/** - * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. - * - * @param filename file to read - * @return closure to use - */ -void * -GNUNET_FS_make_file_reader_context_ (const char *filename); - - - -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -size_t -GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, - char **emsg); - -/** - * Notification of FS that a search probe has made progress. - * This function is used INSTEAD of the client's event handler - * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. - * - * @param cls closure, always NULL (!), actual closure - * is in the client-context of the info struct - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the GNUNET_FS_ProgressInfo struct. - */ -void * -GNUNET_FS_search_probe_progress_ (void *cls, - const struct GNUNET_FS_ProgressInfo *info); - - -/** - * Main function that performs the upload. - * - * @param cls "struct GNUNET_FS_PublishContext" identifies the upload - * @param tc task context - */ -void -GNUNET_FS_publish_main_ (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - -/** - * Function called once the hash of the file - * that is being unindexed has been computed. - * - * @param cls closure, unindex context - * @param file_id computed hash, NULL on error - */ -void -GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); - - -/** - * Fill in all of the generic fields for a publish event and call the - * callback. - * - * @param pi structure to fill in - * @param pc overall publishing context - * @param p file information for the file being published - * @param offset where in the file are we so far - * @return value returned from callback - */ -void * -GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_PublishContext *pc, - const struct GNUNET_FS_FileInformation *p, - uint64_t offset); - - -/** - * Fill in all of the generic fields for a download event and call the - * callback. - * - * @param pi structure to fill in - * @param dc overall download context - */ -void -GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_DownloadContext *dc); - - -/** - * Task that creates the initial (top-level) download - * request for the file. - * - * @param cls the 'struct GNUNET_FS_DownloadContext' - * @param tc scheduler context - */ -void -GNUNET_FS_download_start_task_ (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - - -/** - * Fill in all of the generic fields for - * an unindex event and call the callback. - * - * @param pi structure to fill in - * @param uc overall unindex context - * @param offset where we are in the file (for progress) - */ -void -GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_UnindexContext *uc, - uint64_t offset); - -/** - * Fill in all of the generic fields for a search event and - * call the callback. - * - * @param pi structure to fill in - * @param sc overall search context - * @return value returned by the callback - */ -void * -GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, - struct GNUNET_FS_SearchContext *sc); - - -/** - * Connect to the datastore and remove the blocks. - * - * @param uc context for the unindex operation. - */ -void -GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); - -/** - * Build the request and actually initiate the search using the - * GNUnet FS service. - * - * @param sc search context - * @return GNUNET_OK on success, GNUNET_SYSERR on error - */ -int -GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); - -/** - * Start the downloading process (by entering the queue). - * - * @param dc our download context - */ -void -GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); - - -/** - * Start download probes for the given search result. - * - * @param sr the search result - */ -void -GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); - -/** - * Remove serialization/deserialization file from disk. - * - * @param h master context - * @param ext component of the path - * @param ent entity identifier - */ -void -GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *ent); - - -/** - * Remove serialization/deserialization directory from disk. - * - * @param h master context - * @param ext component of the path - * @param uni unique name of parent - */ -void -GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, - const char *uni); - - -/** - * Synchronize this file-information struct with its mirror - * on disk. Note that all internal FS-operations that change - * file information data should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param fi the struct to sync - */ -void -GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); - -/** - * Synchronize this publishing struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param pc the struct to sync - */ -void -GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); - -/** - * Synchronize this unindex struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param uc the struct to sync - */ -void -GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); - -/** - * Synchronize this search struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sc the struct to sync - */ -void -GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); - -/** - * Synchronize this search result with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param sr the struct to sync - */ -void -GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); - -/** - * Synchronize this download struct with its mirror - * on disk. Note that all internal FS-operations that change - * publishing structs should already call "sync" internally, - * so this function is likely not useful for clients. - * - * @param dc the struct to sync - */ -void -GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); - -/** - * Create SUSPEND event for the given publish operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_PublishContext' to signal for - */ -void -GNUNET_FS_publish_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given search operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_SearchContext' to signal for - */ -void -GNUNET_FS_search_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given download operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for - */ -void -GNUNET_FS_download_signal_suspend_ (void *cls); - -/** - * Create SUSPEND event for the given unindex operation - * and then clean up our state (without stop signal). - * - * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for - */ -void -GNUNET_FS_unindex_signal_suspend_ (void *cls); - -/** - * Function signature of the functions that can be called - * to trigger suspend signals and clean-up for top-level - * activities. - * - * @param cls closure - */ -typedef void (*SuspendSignalFunction) (void *cls); - -/** - * We track all of the top-level activities of FS - * so that we can signal 'suspend' on shutdown. - */ -struct TopLevelActivity -{ - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *next; - - /** - * This is a doubly-linked list. - */ - struct TopLevelActivity *prev; - - /** - * Function to call for suspend-signalling and clean up. - */ - SuspendSignalFunction ssf; - - /** - * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) - */ - void *ssf_cls; -}; - - -/** - * Create a top-level activity entry. - * - * @param h global fs handle - * @param ssf suspend signal function to use - * @param ssf_cls closure for ssf - * @return fresh top-level activity handle - */ -struct TopLevelActivity * -GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, - void *ssf_cls); - - -/** - * Destroy a top-level activity entry. - * - * @param h global fs handle - * @param top top level activity entry - */ -void -GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top); - - - -/** - * Master context for most FS operations. - */ -struct GNUNET_FS_Handle -{ - /** - * Configuration to use. - */ - const struct GNUNET_CONFIGURATION_Handle *cfg; - - /** - * Name of our client. - */ - char *client_name; - - /** - * Function to call with updates on our progress. - */ - GNUNET_FS_ProgressCallback upcb; - - /** - * Closure for upcb. - */ - void *upcb_cls; - - /** - * Head of DLL of top-level activities. - */ - struct TopLevelActivity *top_head; - - /** - * Tail of DLL of top-level activities. - */ - struct TopLevelActivity *top_tail; - - /** - * Head of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_head; - - /** - * Tail of DLL of running jobs. - */ - struct GNUNET_FS_QueueEntry *running_tail; - - /** - * Head of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_head; - - /** - * Tail of DLL of pending jobs. - */ - struct GNUNET_FS_QueueEntry *pending_tail; - - /** - * Task that processes the jobs in the running and pending queues - * (and moves jobs around as needed). - */ - GNUNET_SCHEDULER_TaskIdentifier queue_job; - - /** - * Average time we take for a single request to be satisfied. - * FIXME: not yet calcualted properly... - */ - struct GNUNET_TIME_Relative avg_block_latency; - - /** - * How many actual downloads do we have running right now? - */ - unsigned int active_downloads; - - /** - * How many blocks do the active downloads have? - */ - unsigned int active_blocks; - - /** - * General flags. - */ - enum GNUNET_FS_Flags flags; - - /** - * Maximum number of parallel downloads. - */ - unsigned int max_parallel_downloads; - - /** - * Maximum number of parallel requests. - */ - unsigned int max_parallel_requests; - -}; - - -/** - * Handle for controlling a publication process. - */ -struct GNUNET_FS_PublishContext -{ - /** - * Handle to the global fs context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * File-structure that is being shared. - */ - struct GNUNET_FS_FileInformation *fi; - - /** - * Namespace that we are publishing in, NULL if we have no namespace. - */ - struct GNUNET_FS_Namespace *namespace; - - /** - * ID of the content in the namespace, NULL if we have no namespace. - */ - char *nid; - - /** - * ID for future updates, NULL if we have no namespace or no updates. - */ - char *nuid; - - /** - * Filename used for serializing information about this operation - * (should be determined using 'mktemp'). - */ - char *serialization; - - /** - * Our own client handle for the FS service; only briefly used when - * we start to index a file, otherwise NULL. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Current position in the file-tree for the upload. - */ - struct GNUNET_FS_FileInformation *fi_pos; - - /** - * Non-null if we are currently hashing a file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Connection to the datastore service. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Queue entry for reservation/unreservation. - */ - struct GNUNET_DATASTORE_QueueEntry *qre; - - /** - * ID of the task performing the upload. NO_TASK if the upload has - * completed. - */ - GNUNET_SCHEDULER_TaskIdentifier upload_task; - - /** - * Storage space to reserve for the operation. - */ - uint64_t reserve_space; - - /** - * Overall number of entries to reserve for the - * publish operation. - */ - uint32_t reserve_entries; - - /** - * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is - * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from - * the datastore service (in which case this struct must not be - * freed until we have that response). If someone tries to stop the - * download for good during this period, "in_network_wait" is set to - * GNUNET_SYSERR which will cause the struct to be destroyed right - * after we have the reply (or timeout) from the datastore service. - */ - int in_network_wait; - - /** - * Options for publishing. - */ - enum GNUNET_FS_PublishOptions options; - - /** - * Space reservation ID with datastore service - * for this upload. - */ - int rid; - - /** - * Set to GNUNET_YES if all processing has completed. - */ - int all_done; -}; - - -/** - * Phases of unindex processing (state machine). - */ -enum UnindexState -{ - /** - * We're currently hashing the file. - */ - UNINDEX_STATE_HASHING = 0, - - /** - * We're telling the datastore to delete - * the respective entries. - */ - UNINDEX_STATE_DS_REMOVE = 1, - - /** - * We're notifying the FS service about - * the unindexing. - */ - UNINDEX_STATE_FS_NOTIFY = 2, - - /** - * We're done. - */ - UNINDEX_STATE_COMPLETE = 3, - - /** - * We've encountered a fatal error. - */ - UNINDEX_STATE_ERROR = 4 -}; - - -/** - * Handle for controlling an unindexing operation. - */ -struct GNUNET_FS_UnindexContext -{ - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry. - */ - struct TopLevelActivity *top; - - /** - * Name of the file that we are unindexing. - */ - char *filename; - - /** - * Short name under which we are serializing the state of this operation. - */ - char *serialization; - - /** - * Connection to the FS service, only valid during the - * UNINDEX_STATE_FS_NOTIFY phase. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Connection to the datastore service, only valid during the - * UNINDEX_STATE_DS_NOTIFY phase. - */ - struct GNUNET_DATASTORE_Handle *dsh; - - /** - * Pointer kept for the client. - */ - void *client_info; - - /** - * Merkle-ish tree encoder context. - */ - struct GNUNET_FS_TreeEncoder *tc; - - /** - * Handle used to read the file. - */ - struct GNUNET_DISK_FileHandle *fh; - - /** - * Error message, NULL on success. - */ - char *emsg; - - /** - * Context for hashing of the file. - */ - struct GNUNET_CRYPTO_FileHashContext *fhc; - - /** - * Overall size of the file. - */ - uint64_t file_size; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Hash of the file's contents (once computed). - */ - GNUNET_HashCode file_id; - - /** - * Current operatinonal phase. - */ - enum UnindexState state; - -}; - - -/** - * Information we keep for each keyword in - * a keyword search. - */ -struct SearchRequestEntry -{ - /** - * Hash of the original keyword, also known as the - * key (for decrypting the KBlock). - */ - GNUNET_HashCode key; - - /** - * Hash of the public key, also known as the query. - */ - GNUNET_HashCode query; - - /** - * Map that contains a "struct GNUNET_FS_SearchResult" for each result that - * was found under this keyword. Note that the entries will point - * to the same locations as those in the master result map (in - * "struct GNUNET_FS_SearchContext"), so they should not be freed. - * The key for each entry is the XOR of the key and query in the CHK - * URI (as a unique identifier for the search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *results; - - /** - * Is this keyword a mandatory keyword - * (started with '+')? - */ - int mandatory; - -}; - - -/** - * Handle for controlling a search. - */ -struct GNUNET_FS_SearchContext -{ - /** - * Handle to the global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * List of keywords that we're looking for. - */ - struct GNUNET_FS_Uri *uri; - - /** - * For update-searches, link to the search result that triggered - * the update search; otherwise NULL. - */ - struct GNUNET_FS_SearchResult *psearch_result; - - /** - * Connection to the FS service. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Pointer we keep for the client. - */ - void *client_info; - - /** - * Name of the file on disk we use for persistence. - */ - char *serialization; - - /** - * Error message (non-NULL if this operation failed). - */ - char *emsg; - - /** - * Map that contains a "struct GNUNET_FS_SearchResult" for each result that - * was found in the search. The key for each entry is the XOR of - * the key and query in the CHK URI (as a unique identifier for the - * search result). - */ - struct GNUNET_CONTAINER_MultiHashMap *master_result_map; - - /** - * Per-keyword information for a keyword search. This array will - * have exactly as many entries as there were keywords. - */ - struct SearchRequestEntry *requests; - - /** - * When did we start? - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the search is being stopped (if not - * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service. - */ - GNUNET_SCHEDULER_TaskIdentifier task; - - /** - * How many of the entries in the search request - * map have been passed to the service so far? - */ - unsigned int search_request_map_offset; - - /** - * How many of the keywords in the KSK - * map have been passed to the service so far? - */ - unsigned int keyword_offset; - - /** - * Anonymity level for the search. - */ - uint32_t anonymity; - - /** - * Number of mandatory keywords in this query. - */ - uint32_t mandatory_count; - - /** - * Options for the search. - */ - enum GNUNET_FS_SearchOptions options; -}; - - -/** - * FSM for possible states a block can go through. The typical - * order of progression is linear through the states, alternatives - * are documented in the comments. - */ -enum BlockRequestState -{ - /** - * Initial state, block has only been allocated (since it is - * relevant to the overall download request). - */ - BRS_INIT = 0, - - /** - * We've checked the block on the path down the tree, and the - * content on disk did match the desired CHK, but not all - * the way down, so at the bottom some blocks will still - * need to be reconstructed). - */ - BRS_RECONSTRUCT_DOWN = 1, - - /** - * We've calculated the CHK bottom-up based on the meta data. - * This may work, but if it did we have to write the meta data to - * disk at the end (and we still need to check against the - * CHK set on top). - */ - BRS_RECONSTRUCT_META_UP = 2, - - /** - * We've calculated the CHK bottom-up based on what we have on - * disk, which may not be what the desired CHK is. If the - * reconstructed CHKs match whatever comes from above, we're - * done with the respective subtree. - */ - BRS_RECONSTRUCT_UP = 3, - - /** - * We've determined the real, desired CHK for this block - * (full tree reconstruction failed), request is now pending. - * If the CHK that bubbled up through reconstruction did match - * the top-level request, the state machine for the subtree - * would have moved to BRS_DOWNLOAD_UP. - */ - BRS_CHK_SET = 4, - - /** - * We've successfully downloaded this block, but the children - * still need to be either downloaded or verified (download - * request propagates down). If the download fails, the - * state machine for this block may move to - * BRS_DOWNLOAD_ERROR instead. - */ - BRS_DOWNLOAD_DOWN = 5, - - /** - * This block and all of its children have been downloaded - * successfully (full completion propagates up). - */ - BRS_DOWNLOAD_UP = 6, - - /** - * We got a block back that matched the query but did not hash to - * the key (malicious publisher or hash collision); this block - * can never be downloaded (error propagates up). - */ - BRS_ERROR = 7 -}; - - -/** - * Information about an active download request. - */ -struct DownloadRequest -{ - /** - * While pending, we keep all download requests in a doubly-linked list. - */ - struct DownloadRequest *next; - - /** - * While pending, we keep all download requests in a doubly-linked list. - */ - struct DownloadRequest *prev; - - /** - * Parent in the CHK-tree. - */ - struct DownloadRequest *parent; - - /** - * Array (!) of child-requests, or NULL for the bottom of the tree. - */ - struct DownloadRequest **children; - - /** - * CHK for the request for this block (set during reconstruction - * to what we have on disk, later to what we want to have). - */ - struct ContentHashKey chk; - - /** - * Offset of the corresponding block. Specifically, first (!) byte of - * the first DBLOCK in the subtree induced by block represented by - * this request. - */ - uint64_t offset; - - /** - * Number of entries in 'children' array. - */ - unsigned int num_children; - - /** - * Depth of the corresponding block in the tree. 0==DBLOCKs. - */ - unsigned int depth; - - /** - * State in the FSM. - */ - enum BlockRequestState state; - - /** - * GNUNET_YES if this entry is in the pending list. - */ - int is_pending; - -}; - - -/** - * (recursively) free download request structure - * - * @param dr request to free - */ -void -GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); - - -/** - * Context for controlling a download. - */ -struct GNUNET_FS_DownloadContext -{ - - /** - * Global FS context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Our top-level activity entry (if we are top-level, otherwise NULL). - */ - struct TopLevelActivity *top; - - /** - * Connection to the FS service. - */ - struct GNUNET_CLIENT_Connection *client; - - /** - * Parent download (used when downloading files - * in directories). - */ - struct GNUNET_FS_DownloadContext *parent; - - /** - * Associated search (used when downloading files - * based on search results), or NULL for none. - */ - struct GNUNET_FS_SearchResult *search; - - /** - * Head of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_head; - - /** - * Tail of list of child downloads. - */ - struct GNUNET_FS_DownloadContext *child_tail; - - /** - * Previous download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *prev; - - /** - * Next download belonging to the same parent. - */ - struct GNUNET_FS_DownloadContext *next; - - /** - * Context kept for the client. - */ - void *client_info; - - /** - * URI that identifies the file that we are downloading. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Known meta-data for the file (can be NULL). - */ - struct GNUNET_CONTAINER_MetaData *meta; - - /** - * Error message, NULL if we're doing OK. - */ - char *emsg; - - /** - * Random portion of filename we use for syncing state of this - * download. - */ - char *serialization; - - /** - * Where are we writing the data (name of the - * file, can be NULL!). - */ - char *filename; - - /** - * Where are we writing the data temporarily (name of the - * file, can be NULL!); used if we do not have a permanent - * name and we are a directory and we do a recursive download. - */ - char *temp_filename; - - /** - * Our entry in the job queue. - */ - struct GNUNET_FS_QueueEntry *job_queue; - - /** - * Non-NULL if we are currently having a request for - * transmission pending with the client handle. - */ - struct GNUNET_CLIENT_TransmitHandle *th; - - /** - * Tree encoder used for the reconstruction. - */ - struct GNUNET_FS_TreeEncoder *te; - - /** - * File handle for reading data from an existing file - * (to pass to tree encoder). - */ - struct GNUNET_DISK_FileHandle *rfh; - - /** - * Map of active requests (those waiting for a response). The key - * is the hash of the encryped block (aka query). - */ - struct GNUNET_CONTAINER_MultiHashMap *active; - - /** - * Head of linked list of pending requests. - */ - struct DownloadRequest *pending_head; - - /** - * Head of linked list of pending requests. - */ - struct DownloadRequest *pending_tail; - - /** - * Top-level download request. - */ - struct DownloadRequest *top_request; - - /** - * Identity of the peer having the content, or all-zeros - * if we don't know of such a peer. - */ - struct GNUNET_PeerIdentity target; - - /** - * ID of a task that is using this struct and that must be cancelled - * when the download is being stopped (if not - * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some - * artificial delay when trying to reconnect to the FS service or - * the task processing incrementally the data on disk, or the - * task requesting blocks, etc. - */ - GNUNET_SCHEDULER_TaskIdentifier task; - - /** - * What is the first offset that we're interested - * in? - */ - uint64_t offset; - - /** - * How many bytes starting from offset are desired? - * This is NOT the overall length of the file! - */ - uint64_t length; - - /** - * How many bytes have we already received within - * the specified range (DBlocks only). - */ - uint64_t completed; - - /** - * What was the size of the file on disk that we're downloading - * before we started? Used to detect if there is a point in - * checking an existing block on disk for matching the desired - * content. 0 if the file did not exist already. - */ - uint64_t old_file_size; - - /** - * Time download was started. - */ - struct GNUNET_TIME_Absolute start_time; - - /** - * Desired level of anonymity. - */ - uint32_t anonymity; - - /** - * The depth of the file-tree. - */ - unsigned int treedepth; - - /** - * Options for the download. - */ - enum GNUNET_FS_DownloadOptions options; - - /** - * Flag set upon transitive completion (includes child downloads). - * This flag is only set to GNUNET_YES for directories where all - * child-downloads have also completed (and signalled completion). - */ - int has_finished; - - /** - * Have we started the receive continuation yet? - */ - int in_receive; - -}; - - -/** - * Information about an (updateable) node in the - * namespace. - */ -struct NamespaceUpdateNode -{ - /** - * Identifier for this node. - */ - char *id; - - /** - * Identifier of children of this node. - */ - char *update; - - /** - * Metadata for this entry. - */ - struct GNUNET_CONTAINER_MetaData *md; - - /** - * URI of this entry in the namespace. - */ - struct GNUNET_FS_Uri *uri; - - /** - * Namespace update generation ID. Used to ensure - * freshness of the tree_id. - */ - unsigned int nug; - - /** - * TREE this entry belongs to (if nug is current). - */ - unsigned int tree_id; - -}; - - -struct GNUNET_FS_Namespace -{ - - /** - * Handle to the FS service context. - */ - struct GNUNET_FS_Handle *h; - - /** - * Array with information about nodes in the namespace. - */ - struct NamespaceUpdateNode **update_nodes; - - /** - * Private key for the namespace. - */ - struct GNUNET_CRYPTO_RsaPrivateKey *key; - - /** - * Hash map mapping identifiers of update nodes - * to the update nodes (initialized on-demand). - */ - struct GNUNET_CONTAINER_MultiHashMap *update_map; - - /** - * Name of the file with the private key. - */ - char *filename; - - /** - * Name of the namespace. - */ - char *name; - - /** - * Size of the update nodes array. - */ - unsigned int update_node_count; - - /** - * Reference counter. - */ - unsigned int rc; - - /** - * Generator for unique nug numbers. - */ - unsigned int nug_gen; -}; - - /** * Message sent from a GNUnet (fs) publishing activity to the * gnunet-fs-service to initiate indexing of a file. The service is diff --git a/src/fs/fs.c b/src/fs/fs_api.c similarity index 99% rename from src/fs/fs.c rename to src/fs/fs_api.c index 83c0bf271..ce4b3b887 100644 --- a/src/fs/fs.c +++ b/src/fs/fs_api.c @@ -19,7 +19,7 @@ */ /** - * @file fs/fs.c + * @file fs/fs_api.c * @brief main FS functions (master initialization, serialization, deserialization, shared code) * @author Christian Grothoff */ @@ -27,7 +27,7 @@ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h new file mode 100644 index 000000000..f358047aa --- /dev/null +++ b/src/fs/fs_api.h @@ -0,0 +1,1907 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_api.h + * @brief shared definitions for the FS library + * @author Igor Wronsky, Christian Grothoff + */ +#ifndef FS_API_H +#define FS_API_H + +#include "gnunet_constants.h" +#include "gnunet_datastore_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_fs_service.h" +#include "gnunet_block_lib.h" +#include "block_fs.h" +#include "fs.h" + +/** + * Size of the individual blocks used for file-sharing. + */ +#define DBLOCK_SIZE (32*1024) + +/** + * Pick a multiple of 2 here to achive 8-byte alignment! We also + * probably want DBlocks to have (roughly) the same size as IBlocks. + * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128 + * byte = 2 * 512 bits). DO NOT CHANGE! + */ +#define CHK_PER_INODE 256 + +/** + * Maximum size for a file to be considered for inlining in a + * directory. + */ +#define MAX_INLINE_SIZE 65536 + +/** + * Name of the directory with top-level searches. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" + +/** + * Name of the directory with sub-searches (namespace-updates). + */ +#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" + +/** + * Name of the directory with master downloads (not associated + * with search or part of another download). + */ +#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" + +/** + * Name of the directory with downloads that are part of another + * download or a search. + */ +#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" + +/** + * Name of the directory with publishing operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" + +/** + * Name of the directory with files that are being published + */ +#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" + +/** + * Name of the directory with unindex operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" + + +/** + * @brief complete information needed + * to download a file. + */ +struct FileIdentifier +{ + + /** + * Total size of the file in bytes. (network byte order (!)) + */ + uint64_t file_length; + + /** + * Query and key of the top GNUNET_EC_IBlock. + */ + struct ContentHashKey chk; + +}; + + +/** + * Information about a file and its location + * (peer claiming to share the file). + */ +struct Location +{ + /** + * Information about the shared file. + */ + struct FileIdentifier fi; + + /** + * Identity of the peer sharing the file. + */ + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; + + /** + * Time when this location URI expires. + */ + struct GNUNET_TIME_Absolute expirationTime; + + /** + * RSA signature over the GNUNET_EC_FileIdentifier, + * GNUNET_hash of the peer and expiration time. + */ + struct GNUNET_CRYPTO_RsaSignature contentSignature; + +}; + +/** + * Types of URIs. + */ +enum uri_types +{ + /** + * Content-hash-key (simple file). + */ + chk, + + /** + * Signed key space (file in namespace). + */ + sks, + + /** + * Keyword search key (query with keywords). + */ + ksk, + + /** + * Location (chk with identity of hosting peer). + */ + loc +}; + +/** + * A Universal Resource Identifier (URI), opaque. + */ +struct GNUNET_FS_Uri +{ + /** + * Type of the URI. + */ + enum uri_types type; + + union + { + struct + { + /** + * Keywords start with a '+' if they are + * mandatory (in which case the '+' is NOT + * part of the keyword) and with a + * simple space if they are optional + * (in which case the space is ALSO not + * part of the actual keyword). + * + * Double-quotes to protect spaces and + * %-encoding are NOT used internally + * (only in URI-strings). + */ + char **keywords; + + /** + * Size of the keywords array. + */ + unsigned int keywordCount; + } ksk; + + struct + { + /** + * Hash of the public key for the namespace. + */ + GNUNET_HashCode namespace; + + /** + * Human-readable identifier chosen for this + * entry in the namespace. + */ + char *identifier; + } sks; + + /** + * Information needed to retrieve a file (content-hash-key + * plus file size). + */ + struct FileIdentifier chk; + + /** + * Information needed to retrieve a file including signed + * location (identity of a peer) of the content. + */ + struct Location loc; + } data; + +}; + + +/** + * Information for a file or directory that is + * about to be published. + */ +struct GNUNET_FS_FileInformation +{ + + /** + * Files in a directory are kept as a linked list. + */ + struct GNUNET_FS_FileInformation *next; + + /** + * If this is a file in a directory, "dir" refers to + * the directory; otherwise NULL. + */ + struct GNUNET_FS_FileInformation *dir; + + /** + * Handle to the master context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Metadata to use for the file. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Keywords to use for KBlocks. + */ + struct GNUNET_FS_Uri *keywords; + + /** + * CHK for this file or directory. NULL if + * we have not yet computed it. + */ + struct GNUNET_FS_Uri *chk_uri; + + /** + * Block options for the file. + */ + struct GNUNET_FS_BlockOptions bo; + + /** + * At what time did we start this upload? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Under what filename is this struct serialized + * (for operational persistence). Should be determined + * using 'mktemp'. + */ + char *serialization; + + /** + * Encoder being used to publish this file. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Name of the file or directory (must be an absolute path). + */ + char *filename; + + /** + * Data describing either the file or the directory. + */ + union + { + + /** + * Data for a file. + */ + struct + { + + /** + * Function that can be used to read the data for the file. + */ + GNUNET_FS_DataReader reader; + + /** + * Closure for reader. + */ + void *reader_cls; + + /** + * If this file is being indexed, this value is set to the hash + * over the entire file (when the indexing process is started). + * Otherwise this field is not used. + */ + GNUNET_HashCode file_id; + + /** + * Size of the file (in bytes). + */ + uint64_t file_size; + + /** + * Should the file be indexed or inserted? + */ + int do_index; + + /** + * Is "file_id" already valid? Set to GNUNET_YES once the hash + * has been calculated. + */ + int have_hash; + + /** + * Has the service confirmed our INDEX_START request? + * GNUNET_YES if this step has been completed. + */ + int index_start_confirmed; + + } file; + + /** + * Data for a directory. + */ + struct + { + + /** + * Linked list of entries in the directory. + */ + struct GNUNET_FS_FileInformation *entries; + + /** + * Size of the directory itself (in bytes); 0 if the + * size has not yet been calculated. + */ + size_t dir_size; + + /** + * Pointer to the data for the directory (or NULL if not + * available). + */ + void *dir_data; + + } dir; + + } data; + + /** + * Is this struct for a file or directory? + */ + int is_directory; + + /** + * Are we done publishing this file? + */ + int is_published; + +}; + + +/** + * The job is now ready to run and should use the given client + * handle to communicate with the FS service. + * + * @param cls closure + * @param client handle to use for FS communication + */ +typedef void (*GNUNET_FS_QueueStart) (void *cls, + struct GNUNET_CLIENT_Connection * client); + + +/** + * The job must now stop to run and should destry the client handle as + * soon as possible (ideally prior to returning). + */ +typedef void (*GNUNET_FS_QueueStop) (void *cls); + + +/** + * Entry in the job queue. + */ +struct GNUNET_FS_QueueEntry +{ + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *next; + + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *prev; + + /** + * Function to call when the job is started. + */ + GNUNET_FS_QueueStart start; + + /** + * Function to call when the job needs to stop (or is done / dequeued). + */ + GNUNET_FS_QueueStop stop; + + /** + * Closure for start and stop. + */ + void *cls; + + /** + * Handle to FS primary context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Client handle, or NULL if job is not running. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Time the job was originally queued. + */ + struct GNUNET_TIME_Absolute queue_time; + + /** + * Time the job was started last. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Total amount of time the job has been running (except for the + * current run). + */ + struct GNUNET_TIME_Relative run_time; + + /** + * How many blocks do the active downloads have? + */ + unsigned int blocks; + + /** + * How often have we (re)started this download? + */ + unsigned int start_times; + +}; + + + + +/** + * Information we store for each search result. + */ +struct GNUNET_FS_SearchResult +{ + + /** + * Search context this result belongs to. + */ + struct GNUNET_FS_SearchContext *sc; + + /** + * URI to which this search result refers to. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Metadata for the search result. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Client info for this search result. + */ + void *client_info; + + /** + * ID of a job that is currently probing this results' availability + * (NULL if we are not currently probing). + */ + struct GNUNET_FS_DownloadContext *probe_ctx; + + /** + * ID of an associated download based on this search result (or + * NULL for none). + */ + struct GNUNET_FS_DownloadContext *download; + + /** + * If this search result triggered an update search, this field + * links to the update search. + */ + struct GNUNET_FS_SearchContext *update_search; + + /** + * Name under which this search result is stored on disk. + */ + char *serialization; + + /** + * Key for the search result + */ + GNUNET_HashCode key; + + /** + * ID of the task that will clean up the probe_ctx should it not + * complete on time (and that will need to be cancelled if we clean + * up the search result before then). + */ + GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task; + + /** + * When did the current probe become active? + */ + struct GNUNET_TIME_Absolute probe_active_time; + + /** + * How much longer should we run the current probe before giving up? + */ + struct GNUNET_TIME_Relative remaining_probe_time; + + /** + * Number of mandatory keywords for which we have NOT yet found the + * search result; when this value hits zero, the search result is + * given to the callback. + */ + uint32_t mandatory_missing; + + /** + * Number of optional keywords under which this result was also + * found. + */ + uint32_t optional_support; + + /** + * Number of availability tests that have succeeded for this result. + */ + uint32_t availability_success; + + /** + * Number of availability trials that we have performed for this + * search result. + */ + uint32_t availability_trials; + +}; + + +/** + * Add a job to the queue. + * + * @param h handle to the overall FS state + * @param start function to call to begin the job + * @param stop function to call to pause the job, or on dequeue (if the job was running) + * @param cls closure for start and stop + * @param blocks number of blocks this download has + * @return queue handle + */ +struct GNUNET_FS_QueueEntry * +GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, + GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks); + + +/** + * Dequeue a job from the queue. + * @param qh handle for the job + */ +void +GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh); + + +/** + * Function that provides data by reading from a file. + * + * @param cls closure (points to the file information) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + + +/** + * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. + * + * @param filename file to read + * @return closure to use + */ +void * +GNUNET_FS_make_file_reader_context_ (const char *filename); + + + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + +/** + * Notification of FS that a search probe has made progress. + * This function is used INSTEAD of the client's event handler + * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. + * + * @param cls closure, always NULL (!), actual closure + * is in the client-context of the info struct + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +void * +GNUNET_FS_search_probe_progress_ (void *cls, + const struct GNUNET_FS_ProgressInfo *info); + + +/** + * Main function that performs the upload. + * + * @param cls "struct GNUNET_FS_PublishContext" identifies the upload + * @param tc task context + */ +void +GNUNET_FS_publish_main_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called once the hash of the file + * that is being unindexed has been computed. + * + * @param cls closure, unindex context + * @param file_id computed hash, NULL on error + */ +void +GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); + + +/** + * Fill in all of the generic fields for a publish event and call the + * callback. + * + * @param pi structure to fill in + * @param pc overall publishing context + * @param p file information for the file being published + * @param offset where in the file are we so far + * @return value returned from callback + */ +void * +GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_PublishContext *pc, + const struct GNUNET_FS_FileInformation *p, + uint64_t offset); + + +/** + * Fill in all of the generic fields for a download event and call the + * callback. + * + * @param pi structure to fill in + * @param dc overall download context + */ +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc); + + +/** + * Task that creates the initial (top-level) download + * request for the file. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + * @param tc scheduler context + */ +void +GNUNET_FS_download_start_task_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + + +/** + * Fill in all of the generic fields for + * an unindex event and call the callback. + * + * @param pi structure to fill in + * @param uc overall unindex context + * @param offset where we are in the file (for progress) + */ +void +GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_UnindexContext *uc, + uint64_t offset); + +/** + * Fill in all of the generic fields for a search event and + * call the callback. + * + * @param pi structure to fill in + * @param sc overall search context + * @return value returned by the callback + */ +void * +GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_SearchContext *sc); + + +/** + * Connect to the datastore and remove the blocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Build the request and actually initiate the search using the + * GNUnet FS service. + * + * @param sc search context + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Start the downloading process (by entering the queue). + * + * @param dc our download context + */ +void +GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); + + +/** + * Start download probes for the given search result. + * + * @param sr the search result + */ +void +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier + */ +void +GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent); + + +/** + * Remove serialization/deserialization directory from disk. + * + * @param h master context + * @param ext component of the path + * @param uni unique name of parent + */ +void +GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni); + + +/** + * Synchronize this file-information struct with its mirror + * on disk. Note that all internal FS-operations that change + * file information data should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param fi the struct to sync + */ +void +GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); + +/** + * Synchronize this publishing struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param pc the struct to sync + */ +void +GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); + +/** + * Synchronize this unindex struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param uc the struct to sync + */ +void +GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Synchronize this search struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sc the struct to sync + */ +void +GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Synchronize this search result with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sr the struct to sync + */ +void +GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); + +/** + * Create SUSPEND event for the given publish operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_PublishContext' to signal for + */ +void +GNUNET_FS_publish_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given search operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_SearchContext' to signal for + */ +void +GNUNET_FS_search_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given download operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for + */ +void +GNUNET_FS_download_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given unindex operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for + */ +void +GNUNET_FS_unindex_signal_suspend_ (void *cls); + +/** + * Function signature of the functions that can be called + * to trigger suspend signals and clean-up for top-level + * activities. + * + * @param cls closure + */ +typedef void (*SuspendSignalFunction) (void *cls); + +/** + * We track all of the top-level activities of FS + * so that we can signal 'suspend' on shutdown. + */ +struct TopLevelActivity +{ + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *next; + + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *prev; + + /** + * Function to call for suspend-signalling and clean up. + */ + SuspendSignalFunction ssf; + + /** + * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) + */ + void *ssf_cls; +}; + + +/** + * Create a top-level activity entry. + * + * @param h global fs handle + * @param ssf suspend signal function to use + * @param ssf_cls closure for ssf + * @return fresh top-level activity handle + */ +struct TopLevelActivity * +GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, + void *ssf_cls); + + +/** + * Destroy a top-level activity entry. + * + * @param h global fs handle + * @param top top level activity entry + */ +void +GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top); + + + +/** + * Master context for most FS operations. + */ +struct GNUNET_FS_Handle +{ + /** + * Configuration to use. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Name of our client. + */ + char *client_name; + + /** + * Function to call with updates on our progress. + */ + GNUNET_FS_ProgressCallback upcb; + + /** + * Closure for upcb. + */ + void *upcb_cls; + + /** + * Head of DLL of top-level activities. + */ + struct TopLevelActivity *top_head; + + /** + * Tail of DLL of top-level activities. + */ + struct TopLevelActivity *top_tail; + + /** + * Head of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_head; + + /** + * Tail of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_tail; + + /** + * Head of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_head; + + /** + * Tail of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_tail; + + /** + * Task that processes the jobs in the running and pending queues + * (and moves jobs around as needed). + */ + GNUNET_SCHEDULER_TaskIdentifier queue_job; + + /** + * Average time we take for a single request to be satisfied. + * FIXME: not yet calcualted properly... + */ + struct GNUNET_TIME_Relative avg_block_latency; + + /** + * How many actual downloads do we have running right now? + */ + unsigned int active_downloads; + + /** + * How many blocks do the active downloads have? + */ + unsigned int active_blocks; + + /** + * General flags. + */ + enum GNUNET_FS_Flags flags; + + /** + * Maximum number of parallel downloads. + */ + unsigned int max_parallel_downloads; + + /** + * Maximum number of parallel requests. + */ + unsigned int max_parallel_requests; + +}; + + +/** + * Handle for controlling a publication process. + */ +struct GNUNET_FS_PublishContext +{ + /** + * Handle to the global fs context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * File-structure that is being shared. + */ + struct GNUNET_FS_FileInformation *fi; + + /** + * Namespace that we are publishing in, NULL if we have no namespace. + */ + struct GNUNET_FS_Namespace *namespace; + + /** + * ID of the content in the namespace, NULL if we have no namespace. + */ + char *nid; + + /** + * ID for future updates, NULL if we have no namespace or no updates. + */ + char *nuid; + + /** + * Filename used for serializing information about this operation + * (should be determined using 'mktemp'). + */ + char *serialization; + + /** + * Our own client handle for the FS service; only briefly used when + * we start to index a file, otherwise NULL. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Current position in the file-tree for the upload. + */ + struct GNUNET_FS_FileInformation *fi_pos; + + /** + * Non-null if we are currently hashing a file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Connection to the datastore service. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Queue entry for reservation/unreservation. + */ + struct GNUNET_DATASTORE_QueueEntry *qre; + + /** + * ID of the task performing the upload. NO_TASK if the upload has + * completed. + */ + GNUNET_SCHEDULER_TaskIdentifier upload_task; + + /** + * Storage space to reserve for the operation. + */ + uint64_t reserve_space; + + /** + * Overall number of entries to reserve for the + * publish operation. + */ + uint32_t reserve_entries; + + /** + * Typically GNUNET_NO. Set to GNUNET_YES if "upload_task" is + * GNUNET_SCHEDULER_NO_TASK and we're waiting for a response from + * the datastore service (in which case this struct must not be + * freed until we have that response). If someone tries to stop the + * download for good during this period, "in_network_wait" is set to + * GNUNET_SYSERR which will cause the struct to be destroyed right + * after we have the reply (or timeout) from the datastore service. + */ + int in_network_wait; + + /** + * Options for publishing. + */ + enum GNUNET_FS_PublishOptions options; + + /** + * Space reservation ID with datastore service + * for this upload. + */ + int rid; + + /** + * Set to GNUNET_YES if all processing has completed. + */ + int all_done; +}; + + +/** + * Phases of unindex processing (state machine). + */ +enum UnindexState +{ + /** + * We're currently hashing the file. + */ + UNINDEX_STATE_HASHING = 0, + + /** + * We're telling the datastore to delete + * the respective entries. + */ + UNINDEX_STATE_DS_REMOVE = 1, + + /** + * We're notifying the FS service about + * the unindexing. + */ + UNINDEX_STATE_FS_NOTIFY = 2, + + /** + * We're done. + */ + UNINDEX_STATE_COMPLETE = 3, + + /** + * We've encountered a fatal error. + */ + UNINDEX_STATE_ERROR = 4 +}; + + +/** + * Handle for controlling an unindexing operation. + */ +struct GNUNET_FS_UnindexContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry. + */ + struct TopLevelActivity *top; + + /** + * Name of the file that we are unindexing. + */ + char *filename; + + /** + * Short name under which we are serializing the state of this operation. + */ + char *serialization; + + /** + * Connection to the FS service, only valid during the + * UNINDEX_STATE_FS_NOTIFY phase. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Connection to the datastore service, only valid during the + * UNINDEX_STATE_DS_NOTIFY phase. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Merkle-ish tree encoder context. + */ + struct GNUNET_FS_TreeEncoder *tc; + + /** + * Handle used to read the file. + */ + struct GNUNET_DISK_FileHandle *fh; + + /** + * Error message, NULL on success. + */ + char *emsg; + + /** + * Context for hashing of the file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Overall size of the file. + */ + uint64_t file_size; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Hash of the file's contents (once computed). + */ + GNUNET_HashCode file_id; + + /** + * Current operatinonal phase. + */ + enum UnindexState state; + +}; + + +/** + * Information we keep for each keyword in + * a keyword search. + */ +struct SearchRequestEntry +{ + /** + * Hash of the original keyword, also known as the + * key (for decrypting the KBlock). + */ + GNUNET_HashCode key; + + /** + * Hash of the public key, also known as the query. + */ + GNUNET_HashCode query; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found under this keyword. Note that the entries will point + * to the same locations as those in the master result map (in + * "struct GNUNET_FS_SearchContext"), so they should not be freed. + * The key for each entry is the XOR of the key and query in the CHK + * URI (as a unique identifier for the search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *results; + + /** + * Is this keyword a mandatory keyword + * (started with '+')? + */ + int mandatory; + +}; + + +/** + * Handle for controlling a search. + */ +struct GNUNET_FS_SearchContext +{ + /** + * Handle to the global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * List of keywords that we're looking for. + */ + struct GNUNET_FS_Uri *uri; + + /** + * For update-searches, link to the search result that triggered + * the update search; otherwise NULL. + */ + struct GNUNET_FS_SearchResult *psearch_result; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Pointer we keep for the client. + */ + void *client_info; + + /** + * Name of the file on disk we use for persistence. + */ + char *serialization; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found in the search. The key for each entry is the XOR of + * the key and query in the CHK URI (as a unique identifier for the + * search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *master_result_map; + + /** + * Per-keyword information for a keyword search. This array will + * have exactly as many entries as there were keywords. + */ + struct SearchRequestEntry *requests; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the search is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * How many of the entries in the search request + * map have been passed to the service so far? + */ + unsigned int search_request_map_offset; + + /** + * How many of the keywords in the KSK + * map have been passed to the service so far? + */ + unsigned int keyword_offset; + + /** + * Anonymity level for the search. + */ + uint32_t anonymity; + + /** + * Number of mandatory keywords in this query. + */ + uint32_t mandatory_count; + + /** + * Options for the search. + */ + enum GNUNET_FS_SearchOptions options; +}; + + +/** + * FSM for possible states a block can go through. The typical + * order of progression is linear through the states, alternatives + * are documented in the comments. + */ +enum BlockRequestState +{ + /** + * Initial state, block has only been allocated (since it is + * relevant to the overall download request). + */ + BRS_INIT = 0, + + /** + * We've checked the block on the path down the tree, and the + * content on disk did match the desired CHK, but not all + * the way down, so at the bottom some blocks will still + * need to be reconstructed). + */ + BRS_RECONSTRUCT_DOWN = 1, + + /** + * We've calculated the CHK bottom-up based on the meta data. + * This may work, but if it did we have to write the meta data to + * disk at the end (and we still need to check against the + * CHK set on top). + */ + BRS_RECONSTRUCT_META_UP = 2, + + /** + * We've calculated the CHK bottom-up based on what we have on + * disk, which may not be what the desired CHK is. If the + * reconstructed CHKs match whatever comes from above, we're + * done with the respective subtree. + */ + BRS_RECONSTRUCT_UP = 3, + + /** + * We've determined the real, desired CHK for this block + * (full tree reconstruction failed), request is now pending. + * If the CHK that bubbled up through reconstruction did match + * the top-level request, the state machine for the subtree + * would have moved to BRS_DOWNLOAD_UP. + */ + BRS_CHK_SET = 4, + + /** + * We've successfully downloaded this block, but the children + * still need to be either downloaded or verified (download + * request propagates down). If the download fails, the + * state machine for this block may move to + * BRS_DOWNLOAD_ERROR instead. + */ + BRS_DOWNLOAD_DOWN = 5, + + /** + * This block and all of its children have been downloaded + * successfully (full completion propagates up). + */ + BRS_DOWNLOAD_UP = 6, + + /** + * We got a block back that matched the query but did not hash to + * the key (malicious publisher or hash collision); this block + * can never be downloaded (error propagates up). + */ + BRS_ERROR = 7 +}; + + +/** + * Information about an active download request. + */ +struct DownloadRequest +{ + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *next; + + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *prev; + + /** + * Parent in the CHK-tree. + */ + struct DownloadRequest *parent; + + /** + * Array (!) of child-requests, or NULL for the bottom of the tree. + */ + struct DownloadRequest **children; + + /** + * CHK for the request for this block (set during reconstruction + * to what we have on disk, later to what we want to have). + */ + struct ContentHashKey chk; + + /** + * Offset of the corresponding block. Specifically, first (!) byte of + * the first DBLOCK in the subtree induced by block represented by + * this request. + */ + uint64_t offset; + + /** + * Number of entries in 'children' array. + */ + unsigned int num_children; + + /** + * Depth of the corresponding block in the tree. 0==DBLOCKs. + */ + unsigned int depth; + + /** + * State in the FSM. + */ + enum BlockRequestState state; + + /** + * GNUNET_YES if this entry is in the pending list. + */ + int is_pending; + +}; + + +/** + * (recursively) free download request structure + * + * @param dr request to free + */ +void +GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); + + +/** + * Context for controlling a download. + */ +struct GNUNET_FS_DownloadContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Parent download (used when downloading files + * in directories). + */ + struct GNUNET_FS_DownloadContext *parent; + + /** + * Associated search (used when downloading files + * based on search results), or NULL for none. + */ + struct GNUNET_FS_SearchResult *search; + + /** + * Head of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_head; + + /** + * Tail of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_tail; + + /** + * Previous download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *prev; + + /** + * Next download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *next; + + /** + * Context kept for the client. + */ + void *client_info; + + /** + * URI that identifies the file that we are downloading. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Known meta-data for the file (can be NULL). + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Error message, NULL if we're doing OK. + */ + char *emsg; + + /** + * Random portion of filename we use for syncing state of this + * download. + */ + char *serialization; + + /** + * Where are we writing the data (name of the + * file, can be NULL!). + */ + char *filename; + + /** + * Where are we writing the data temporarily (name of the + * file, can be NULL!); used if we do not have a permanent + * name and we are a directory and we do a recursive download. + */ + char *temp_filename; + + /** + * Our entry in the job queue. + */ + struct GNUNET_FS_QueueEntry *job_queue; + + /** + * Non-NULL if we are currently having a request for + * transmission pending with the client handle. + */ + struct GNUNET_CLIENT_TransmitHandle *th; + + /** + * Tree encoder used for the reconstruction. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * File handle for reading data from an existing file + * (to pass to tree encoder). + */ + struct GNUNET_DISK_FileHandle *rfh; + + /** + * Map of active requests (those waiting for a response). The key + * is the hash of the encryped block (aka query). + */ + struct GNUNET_CONTAINER_MultiHashMap *active; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_head; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_tail; + + /** + * Top-level download request. + */ + struct DownloadRequest *top_request; + + /** + * Identity of the peer having the content, or all-zeros + * if we don't know of such a peer. + */ + struct GNUNET_PeerIdentity target; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the download is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service or + * the task processing incrementally the data on disk, or the + * task requesting blocks, etc. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * What is the first offset that we're interested + * in? + */ + uint64_t offset; + + /** + * How many bytes starting from offset are desired? + * This is NOT the overall length of the file! + */ + uint64_t length; + + /** + * How many bytes have we already received within + * the specified range (DBlocks only). + */ + uint64_t completed; + + /** + * What was the size of the file on disk that we're downloading + * before we started? Used to detect if there is a point in + * checking an existing block on disk for matching the desired + * content. 0 if the file did not exist already. + */ + uint64_t old_file_size; + + /** + * Time download was started. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Desired level of anonymity. + */ + uint32_t anonymity; + + /** + * The depth of the file-tree. + */ + unsigned int treedepth; + + /** + * Options for the download. + */ + enum GNUNET_FS_DownloadOptions options; + + /** + * Flag set upon transitive completion (includes child downloads). + * This flag is only set to GNUNET_YES for directories where all + * child-downloads have also completed (and signalled completion). + */ + int has_finished; + + /** + * Have we started the receive continuation yet? + */ + int in_receive; + +}; + + +/** + * Information about an (updateable) node in the + * namespace. + */ +struct NamespaceUpdateNode +{ + /** + * Identifier for this node. + */ + char *id; + + /** + * Identifier of children of this node. + */ + char *update; + + /** + * Metadata for this entry. + */ + struct GNUNET_CONTAINER_MetaData *md; + + /** + * URI of this entry in the namespace. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Namespace update generation ID. Used to ensure + * freshness of the tree_id. + */ + unsigned int nug; + + /** + * TREE this entry belongs to (if nug is current). + */ + unsigned int tree_id; + +}; + + +struct GNUNET_FS_Namespace +{ + + /** + * Handle to the FS service context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Array with information about nodes in the namespace. + */ + struct NamespaceUpdateNode **update_nodes; + + /** + * Private key for the namespace. + */ + struct GNUNET_CRYPTO_RsaPrivateKey *key; + + /** + * Hash map mapping identifiers of update nodes + * to the update nodes (initialized on-demand). + */ + struct GNUNET_CONTAINER_MultiHashMap *update_map; + + /** + * Name of the file with the private key. + */ + char *filename; + + /** + * Name of the namespace. + */ + char *name; + + /** + * Size of the update nodes array. + */ + unsigned int update_node_count; + + /** + * Reference counter. + */ + unsigned int rc; + + /** + * Generator for unique nug numbers. + */ + unsigned int nug_gen; +}; + + +#endif + +/* end of fs_api.h */ diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index f43fe14a7..d5f73623a 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c @@ -34,7 +34,7 @@ */ #include "platform.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /** * String that is used to indicate that a file diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index af889d804..09785f6dd 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -28,7 +28,7 @@ #include "platform.h" #include "gnunet_constants.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_DOWNLOAD GNUNET_EXTRA_LOGGING diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index bb84b948e..b094ed7d5 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -30,7 +30,7 @@ #include "platform.h" #include #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c index b50d0624b..03747742b 100644 --- a/src/fs/fs_getopt.c +++ b/src/fs/fs_getopt.c @@ -25,7 +25,7 @@ */ #include "platform.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /* ******************** command-line option parsing API ******************** */ diff --git a/src/fs/fs_list_indexed.c b/src/fs/fs_list_indexed.c index 85e12db32..0896de347 100644 --- a/src/fs/fs_list_indexed.c +++ b/src/fs/fs_list_indexed.c @@ -28,7 +28,7 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c index 6bd4e59f8..d4afc0d55 100644 --- a/src/fs/fs_misc.c +++ b/src/fs/fs_misc.c @@ -25,7 +25,7 @@ #include "platform.h" #include "gnunet_constants.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c index 47ca6bad1..9d754009b 100644 --- a/src/fs/fs_namespace.c +++ b/src/fs/fs_namespace.c @@ -28,10 +28,22 @@ #include "gnunet_signatures.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define DEBUG_NAMESPACE GNUNET_EXTRA_LOGGING + +/** + * Maximum legal size for an sblock. + */ +#define MAX_SBLOCK_SIZE (60 * 1024) + +/** + * Maximum legal size for an nblock. + */ +#define MAX_NBLOCK_SIZE (60 * 1024) + + /** * Return the name of the directory in which we store * our local namespaces (or rather, their public keys). diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 4f27abae4..9a834ee86 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -31,11 +31,16 @@ #include "gnunet_signatures.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_PUBLISH GNUNET_EXTRA_LOGGING +/** + * Maximum legal size for a kblock. + */ +#define MAX_KBLOCK_SIZE (60 * 1024) + /** * Fill in all of the generic fields for diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c index 93a787477..b5152cc66 100644 --- a/src/fs/fs_search.c +++ b/src/fs/fs_search.c @@ -28,10 +28,15 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" #define DEBUG_SEARCH GNUNET_EXTRA_LOGGING +/** + * Number of availability trials we perform per search result. + */ +#define AVAILABILITY_TRIALS_MAX 8 + /** * Fill in all of the generic fields for a search event and * call the callback. diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c index 1c15c6f60..06ab01f8d 100644 --- a/src/fs/fs_test_lib.c +++ b/src/fs/fs_test_lib.c @@ -27,7 +27,7 @@ * @author Christian Grothoff */ #include "platform.h" -#include "fs.h" +#include "fs_api.h" #include "fs_test_lib.h" #include "gnunet_testing_lib.h" diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h index 56c32a390..5b1c2024f 100644 --- a/src/fs/fs_tree.h +++ b/src/fs/fs_tree.h @@ -32,7 +32,7 @@ #ifndef GNUNET_FS_TREE_H #define GNUNET_FS_TREE_H -#include "fs.h" +#include "fs_api.h" /** * Compute the depth of the CHK tree. diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index 8dab62051..79b7e8c73 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -28,7 +28,7 @@ #include "gnunet_constants.h" #include "gnunet_fs_service.h" #include "gnunet_protocols.h" -#include "fs.h" +#include "fs_api.h" #include "fs_tree.h" #define DEBUG_UNINDEX GNUNET_EXTRA_LOGGING diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c index 48fabc1fb..62fd51304 100644 --- a/src/fs/fs_uri.c +++ b/src/fs/fs_uri.c @@ -81,7 +81,7 @@ #include "platform.h" #include "gnunet_fs_service.h" #include "gnunet_signatures.h" -#include "fs.h" +#include "fs_api.h" /** diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index daffce444..48b7bca29 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -25,8 +25,6 @@ * * To use: * - consider re-issue GSF_dht_lookup_ after non-DHT reply received - * - implement 'SUPPORT_DELAYS' - * */ #include "platform.h" #include diff --git a/src/fs/gnunet-service-fs.h b/src/fs/gnunet-service-fs.h index e4efbb8f5..7efeedd5f 100644 --- a/src/fs/gnunet-service-fs.h +++ b/src/fs/gnunet-service-fs.h @@ -37,26 +37,13 @@ #define DEBUG_FS_CLIENT GNUNET_EXTRA_LOGGING - /** - * Should we introduce random latency in processing? Required for proper - * implementation of GAP, but can be disabled for performance evaluation of - * the basic routing algorithm. - * - * Note that with delays enabled, performance can be significantly lower - * (several orders of magnitude in 2-peer test runs); if you want to - * measure throughput of other components, set this to NO. Also, you - * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for - * a rather wasteful mode of operation (that might still get the highest - * throughput overall). - * - * Performance measurements (for 50 MB file, 2 peers): - * - * - Without delays: 3300 kb/s - * - With delays: 101 kb/s + * By which amount do we decrement the TTL for simple forwarding / + * indirection of the query; in milli-seconds. Set somewhat in + * accordance to your network latency (above the time it'll take you + * to send a packet and get a reply). */ -#define SUPPORT_DELAYS GNUNET_NO - +#define TTL_DECREMENT 5000 /** * At what frequency should our datastore load decrease diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c index d4c81d509..9d2385378 100644 --- a/src/fs/gnunet-service-fs_cp.c +++ b/src/fs/gnunet-service-fs_cp.c @@ -32,6 +32,14 @@ #include "gnunet-service-fs_pr.h" #include "gnunet-service-fs_push.h" + +/** + * Ratio for moving average delay calculation. The previous + * average goes in with a factor of (n-1) into the calculation. + * Must be > 0. + */ +#define RUNAVG_DELAY_N 16 + /** * How often do we flush trust values to disk? */ diff --git a/src/fs/gnunet-service-fs_cp.h b/src/fs/gnunet-service-fs_cp.h index a2cc5232c..11be4c6f1 100644 --- a/src/fs/gnunet-service-fs_cp.h +++ b/src/fs/gnunet-service-fs_cp.h @@ -30,6 +30,36 @@ #include "gnunet-service-fs.h" +/** + * Maximum number of outgoing messages we queue per peer. + * + * Performance measurements for 2 peer setup for 50 MB file + * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): + * + * 2: 1700 kb/s, 1372 kb/s + * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s + * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s + * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s + * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s + * + * Conclusion: 16 seems to be a pretty good value (stable + * and high performance, no excessive memory use). + */ +#define MAX_QUEUE_PER_PEER 16 + +/** + * Length of the P2P success tracker. Note that having a very long + * list can also hurt performance. + */ +#define P2P_SUCCESS_LIST_SIZE 8 + +/** + * Length of the CS-2-P success tracker. Note that + * having a very long list can also hurt performance. + */ +#define CS2P_SUCCESS_LIST_SIZE 8 + + /** * Performance data kept for a peer. */ diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c index ac183a980..3b43b4c5f 100644 --- a/src/fs/gnunet-service-fs_pr.c +++ b/src/fs/gnunet-service-fs_pr.c @@ -31,6 +31,21 @@ #include "gnunet-service-fs_pe.h" #include "gnunet-service-fs_pr.h" +/** + * Maximum size of the datastore queue for P2P operations. Needs to + * be large enough to queue MAX_QUEUE_PER_PEER operations for roughly + * the number of active (connected) peers. + */ +#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) + +/** + * Bandwidth value of a 0-priority content (must be fairly high + * compared to query since content is typically significantly larger + * -- and more valueable since it can take many queries to get one + * piece of content). + */ +#define CONTENT_BANDWIDTH_VALUE 800 + /** * Hard limit on the number of results we may get from the datastore per query. */ diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c index 515446117..4fde823a1 100644 --- a/src/fs/gnunet-service-fs_push.c +++ b/src/fs/gnunet-service-fs_push.c @@ -33,6 +33,17 @@ #define DEBUG_FS_MIGRATION GNUNET_EXTRA_LOGGING +/** + * Maximum number of blocks we keep in memory for migration. + */ +#define MAX_MIGRATION_QUEUE 8 + +/** + * Blocks are at most migrated to this number of peers + * plus one, each time they are fetched from the database. + */ +#define MIGRATION_LIST_SIZE 2 + /** * How long must content remain valid for us to consider it for migration? * If content will expire too soon, there is clearly no point in pushing diff --git a/src/fs/test_fs_directory.c b/src/fs/test_fs_directory.c index ac581bd6f..81012e764 100644 --- a/src/fs/test_fs_directory.c +++ b/src/fs/test_fs_directory.c @@ -28,7 +28,7 @@ #include #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } diff --git a/src/fs/test_fs_uri.c b/src/fs/test_fs_uri.c index 693bff1d6..72744351b 100644 --- a/src/fs/test_fs_uri.c +++ b/src/fs/test_fs_uri.c @@ -27,7 +27,7 @@ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_fs_service.h" -#include "fs.h" +#include "fs_api.h" #define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } -- 2.25.1