From e67c5886c645a5fda7753d3f72f62ea655d6655b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Fri, 30 Apr 2010 13:41:07 +0000 Subject: [PATCH] fixes --- TODO | 3 +- src/fs/fs.c | 59 +++++++++++++++++++++++++++++++++--- src/fs/fs.h | 24 +++++++++++++++ src/fs/fs_file_information.c | 37 +--------------------- src/include/gnunet_common.h | 16 +++++----- src/util/common_allocation.c | 32 +++++++++++++------ 6 files changed, 112 insertions(+), 59 deletions(-) diff --git a/TODO b/TODO index e65b3b940..0b24b0347 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,7 @@ 0.9.0pre1: * FS: [CG] - - persistence support (publish, unindex, search, download) + - persistence support (unindex, search, download) + - persistence testing (publish) - gnunet-service-fs (hot-path routing, load-based routing, nitpicks) - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is never used diff --git a/src/fs/fs.c b/src/fs/fs.c index e7a52d235..062870cd6 100644 --- a/src/fs/fs.c +++ b/src/fs/fs.c @@ -324,6 +324,42 @@ GNUNET_FS_make_file_reader_context_ (const char *filename) } +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, + uint64_t offset, + size_t max, + void *buf, + char **emsg) +{ + char *data = cls; + + if (max == 0) + { + GNUNET_free_non_null (data); + return 0; + } + memcpy (buf, &data[offset], max); + return max; +} + + + /** * Return the full filename where we would store state information * (for serialization/deserialization). @@ -525,11 +561,24 @@ deserialize_fi_node (struct GNUNET_FS_Handle *h, ret->data.file.do_index = GNUNET_NO; ret->data.file.have_hash = GNUNET_NO; ret->data.file.index_start_confirmed = GNUNET_NO; - /* FIXME: what's our approach for dealing with the - 'reader' and 'reader_cls' fields? I guess the only - good way would be to dump "small" files into - 'rh' and to not support serialization of "large" - files (!?) */ + if (GNUNET_NO == ret->is_published) + { + if (NULL == ret->filename) + { + ret->data.file.reader = &GNUNET_FS_data_reader_copy_; + ret->data.file.reader_cls = GNUNET_malloc_large (ret->data.file.file_size); + if (ret->data.file.reader_cls == NULL) + goto cleanup; + if (GNUNET_OK != + GNUNET_BIO_read (rh, "file-data", ret->data.file.reader_cls, ret->data.file.file_size)) + goto cleanup; + } + else + { + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = GNUNET_FS_make_file_reader_context_ (ret->filename); + } + } break; case 1: /* file-index, no hash */ if (NULL == ret->filename) diff --git a/src/fs/fs.h b/src/fs/fs.h index 953e1e51b..2e870bb3b 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -602,6 +602,30 @@ void * GNUNET_FS_make_file_reader_context_ (const char *filename); + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_(void *cls, + uint64_t offset, + size_t max, + void *buf, + char **emsg); + /** * Notification of FS that a search probe has made progress. * This function is used INSTEAD of the client's event handler diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index df5593f75..df53bcc1c 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -130,41 +130,6 @@ GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h, } -/** - * Function that provides data by copying from a buffer. - * - * @param cls closure (points to the buffer) - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually "max", 0 on error - */ -static size_t -data_reader_copy(void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg) -{ - char *data = cls; - - if (max == 0) - { - GNUNET_free (data); - return 0; - } - memcpy (buf, &data[offset], max); - return max; -} - - /** * Create an entry for a file in a publish-structure. * @@ -205,7 +170,7 @@ GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h, return GNUNET_FS_file_information_create_from_reader (h, client_info, length, - &data_reader_copy, + &GNUNET_FS_data_reader_copy_, data, keywords, meta, diff --git a/src/include/gnunet_common.h b/src/include/gnunet_common.h index d7842a519..cb2ed4c0c 100644 --- a/src/include/gnunet_common.h +++ b/src/include/gnunet_common.h @@ -322,7 +322,7 @@ unsigned long long GNUNET_htonll (unsigned long long n); * * @param size the number of bytes to allocate, must be * smaller than 40 MB. - * @return pointer to size bytes of memory + * @return pointer to size bytes of memory, never NULL (!) */ #define GNUNET_malloc(size) GNUNET_xmalloc_(size, __FILE__, __LINE__) @@ -331,7 +331,7 @@ unsigned long long GNUNET_htonll (unsigned long long n); * The memory will be zero'ed out. * * @param size the number of bytes to allocate - * @return pointer to size bytes of memory + * @return pointer to size bytes of memory, NULL if we do not have enough memory */ #define GNUNET_malloc_large(size) GNUNET_xmalloc_unchecked_(size, __FILE__, __LINE__) @@ -452,16 +452,16 @@ void *GNUNET_xmalloc_ (size_t size, const char *filename, int linenumber); /** - * Allocate memory. This function does not check if the - * allocation request is within reasonable bounds, allowing - * allocations larger than 40 MB. If you don't expect the - * possibility of very large allocations, use GNUNET_malloc instead. - * The memory will be zero'ed out. + * Allocate memory. This function does not check if the allocation + * request is within reasonable bounds, allowing allocations larger + * than 40 MB. If you don't expect the possibility of very large + * allocations, use GNUNET_malloc instead. The memory will be zero'ed + * out. * * @param size number of bytes to allocate * @param filename where is this call being made (for debugging) * @param linenumber line where this call is being made (for debugging) - * @return allocated memory, never NULL + * @return pointer to size bytes of memory, NULL if we do not have enough memory */ void *GNUNET_xmalloc_unchecked_ (size_t size, const char *filename, int linenumber); diff --git a/src/util/common_allocation.c b/src/util/common_allocation.c index 5be7caaa7..e62c12d08 100644 --- a/src/util/common_allocation.c +++ b/src/util/common_allocation.c @@ -47,19 +47,36 @@ static LONG mem_used = 0; * this function (or GNUNET_malloc) to allocate more than several MB * of memory, if you are possibly needing a very large chunk use * GNUNET_xmalloc_unchecked_ instead. - * @param filename where in the code was the call to GNUNET_array_grow - * @param linenumber where in the code was the call to GNUNET_array_grow + * @param filename where in the code was the call to GNUNET_malloc + * @param linenumber where in the code was the call to GNUNET_malloc * @return pointer to size bytes of memory */ void * GNUNET_xmalloc_ (size_t size, const char *filename, int linenumber) { + void *ret; /* As a security precaution, we generally do not allow very large allocations using the default 'GNUNET_malloc' macro */ GNUNET_assert_at (size <= GNUNET_MAX_MALLOC_CHECKED, filename, linenumber); - return GNUNET_xmalloc_unchecked_ (size, filename, linenumber); + ret = GNUNET_xmalloc_unchecked_ (size, filename, linenumber); + if (ret == NULL) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc"); + abort (); + } + return ret; } + +/** + * Wrapper around malloc. Allocates size bytes of memory. + * The memory will be zero'ed out. + * + * @param size the number of bytes to allocate + * @param filename where in the code was the call to GNUNET_malloc_large + * @param linenumber where in the code was the call to GNUNET_malloc_large + * @return pointer to size bytes of memory, NULL if we do not have enough memory + */ void * GNUNET_xmalloc_unchecked_ (size_t size, const char *filename, int linenumber) { @@ -74,10 +91,7 @@ GNUNET_xmalloc_unchecked_ (size_t size, const char *filename, int linenumber) GNUNET_assert_at (size < INT_MAX, filename, linenumber); result = malloc (size); if (result == NULL) - { - GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc"); - abort (); - } + return NULL; memset (result, 0, size); #ifdef W32_MEM_LIMIT @@ -148,8 +162,8 @@ GNUNET_xfree_ (void *ptr, const char *filename, int linenumber) * Dup a string (same semantics as strdup). * * @param str the string to dup - * @param filename where in the code was the call to GNUNET_array_grow - * @param linenumber where in the code was the call to GNUNET_array_grow + * @param filename where in the code was the call to GNUNET_strdup + * @param linenumber where in the code was the call to GNUNET_strdup * @return strdup(str) */ char * -- 2.25.1