* @author Christian Grothoff
*
* TODO:
- * - add support for embedded file data (use padding room!)
- * - add directory builder API to gnunet_fs_service
* - modify directory builder API to support incremental
* generation of directories (to allow directories that
* would not fit into memory to be created)
#include "gnunet_fs_service.h"
#include "fs.h"
+/**
+ * String that is used to indicate that a file
+ * is a GNUnet directory.
+ */
+#define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
+
/**
* Does the meta-data claim that this is a directory?
char *mime;
int ret;
- mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE);
+ if (NULL == md)
+ return GNUNET_SYSERR;
+ mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
if (mime == NULL)
return GNUNET_SYSERR;
ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO;
{
char *mime;
- mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE);
+ mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
if (mime != NULL)
{
GNUNET_break (0 == strcmp (mime,
return;
}
GNUNET_CONTAINER_meta_data_insert (md,
- EXTRACTOR_MIMETYPE,
- GNUNET_FS_DIRECTORY_MIME);
+ "<gnunet>",
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ GNUNET_FS_DIRECTORY_MIME,
+ strlen (GNUNET_FS_DIRECTORY_MIME)+1);
+}
+
+
+/**
+ * Closure for 'find_full_data'.
+ */
+struct GetFullDataClosure
+{
+
+ /**
+ * Extracted binary meta data.
+ */
+ void *data;
+
+ /**
+ * Number of bytes stored in data.
+ */
+ size_t size;
+};
+
+
+/**
+ * Type of a function that libextractor calls for each
+ * meta data item found.
+ *
+ * @param cls closure (user-defined)
+ * @param plugin_name name of the plugin that produced this value;
+ * special values can be used (i.e. '<zlib>' for zlib being
+ * used in the main libextractor library and yielding
+ * meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ * can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return 0 to continue extracting, 1 to abort
+ */
+static int
+find_full_data (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct GetFullDataClosure *gfdc = cls;
+
+ if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
+ {
+ gfdc->size = data_len;
+ if (data_len > 0)
+ {
+ gfdc->data = GNUNET_malloc (data_len);
+ memcpy (gfdc->data,
+ data,
+ data_len);
+ }
+ return 1;
+ }
+ return 0;
}
GNUNET_FS_DirectoryEntryProcessor dep,
void *dep_cls)
{
+ struct GetFullDataClosure full_data;
const char *cdata = data;
char *emsg;
uint64_t pos;
/* URI is never empty, must be end of block,
skip to next alignment */
align =
- ((pos / GNUNET_FS_DBLOCK_SIZE) + 1) * GNUNET_FS_DBLOCK_SIZE;
+ ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
if (align == pos)
{
/* if we were already aligned, still skip a block! */
- align += GNUNET_FS_DBLOCK_SIZE;
+ align += DBLOCK_SIZE;
}
pos = align;
if (pos >= size)
return; /* malformed ! */
}
pos += mdSize;
- /* FIXME: add support for embedded data */
filename = GNUNET_CONTAINER_meta_data_get_by_type (md,
- EXTRACTOR_FILENAME);
+ EXTRACTOR_METATYPE_FILENAME);
+ full_data.size = 0;
+ full_data.data = NULL;
+ GNUNET_CONTAINER_meta_data_iterate (md,
+ &find_full_data,
+ &full_data);
if (dep != NULL)
- dep (dep_cls,
- filename,
- uri,
- md,
- 0,
- NULL);
+ {
+ dep (dep_cls,
+ filename,
+ uri,
+ md,
+ full_data.size,
+ full_data.data);
+ }
+ GNUNET_free_non_null (full_data.data);
GNUNET_free_non_null (filename);
GNUNET_CONTAINER_meta_data_destroy (md);
GNUNET_FS_uri_destroy (uri);
}
}
+/**
+ * Entries in the directory (builder).
+ */
+struct BuilderEntry
+{
+ /**
+ * This is a linked list.
+ */
+ struct BuilderEntry *next;
+
+ /**
+ * Length of this entry.
+ */
+ size_t len;
+};
-void
-GNUNET_FS_directory_create ()
+/**
+ * Internal state of a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder
{
+ /**
+ * Meta-data for the directory itself.
+ */
+ struct GNUNET_CONTAINER_MetaData *meta;
+
+ /**
+ * Head of linked list of entries.
+ */
+ struct BuilderEntry *head;
+
+ /**
+ * Number of entires in the directory.
+ */
+ unsigned int count;
+};
+
+
+/**
+ * Create a directory builder.
+ *
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData *mdir)
+{
+ struct GNUNET_FS_DirectoryBuilder *ret;
+
+ ret = GNUNET_malloc(sizeof(struct GNUNET_FS_DirectoryBuilder));
+ if (mdir != NULL)
+ ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir);
+ else
+ ret->meta = GNUNET_CONTAINER_meta_data_create ();
+ GNUNET_FS_meta_data_make_directory (ret->meta);
+ return ret;
}
-#if 0
+/**
+ * Add an entry to a directory.
+ *
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ * data must point to exactly the number of bytes specified
+ * by the uri which must be of type LOC or CHK
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+ const struct GNUNET_FS_Uri *uri,
+ const struct GNUNET_CONTAINER_MetaData *md,
+ const void *data)
+{
+ struct GNUNET_FS_Uri *curi;
+ struct BuilderEntry *e;
+ uint64_t fsize;
+ uint32_t big;
+ ssize_t ret;
+ size_t mds;
+ size_t mdxs;
+ char *uris;
+ char *ser;
+ char *sptr;
+ size_t slen;
+ struct GNUNET_CONTAINER_MetaData *meta;
+ const struct GNUNET_CONTAINER_MetaData *meta_use;
+
+ GNUNET_assert (! GNUNET_FS_uri_test_ksk (uri));
+ if (NULL != data)
+ {
+ GNUNET_assert (! GNUNET_FS_uri_test_sks (uri));
+ if (GNUNET_FS_uri_test_chk (uri))
+ {
+ fsize = GNUNET_FS_uri_chk_get_file_size (uri);
+ }
+ else
+ {
+ curi = GNUNET_FS_uri_loc_get_uri (uri);
+ GNUNET_assert (NULL != curi);
+ fsize = GNUNET_FS_uri_chk_get_file_size (curi);
+ GNUNET_FS_uri_destroy (curi);
+ }
+ }
+ else
+ {
+ fsize = 0; /* not given */
+ }
+ if (fsize > MAX_INLINE_SIZE)
+ fsize = 0; /* too large */
+ uris = GNUNET_FS_uri_to_string (uri);
+ slen = strlen (uris) + 1;
+ mds =
+ GNUNET_CONTAINER_meta_data_get_serialized_size (md);
+ meta_use = md;
+ meta = NULL;
+ if (fsize > 0)
+ {
+ meta = GNUNET_CONTAINER_meta_data_duplicate (md);
+ GNUNET_CONTAINER_meta_data_insert (meta,
+ "<gnunet>",
+ EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
+ EXTRACTOR_METAFORMAT_BINARY,
+ NULL,
+ data,
+ fsize);
+ mdxs =
+ GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
+ if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
+ (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE)
+ {
+ /* adding full data would not cause us to cross
+ additional blocks, so add it! */
+ meta_use = meta;
+ mds = mdxs;
+ }
+ }
+
+ if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
+ mds = GNUNET_MAX_MALLOC_CHECKED / 2;
+ e = GNUNET_malloc (sizeof(struct BuilderEntry) +
+ slen + mds + sizeof (uint32_t));
+ ser = (char*) &e[1];
+ memcpy (ser, uris, slen);
+ GNUNET_free (uris);
+ sptr = &ser[slen + sizeof(uint32_t)];
+ ret = GNUNET_CONTAINER_meta_data_serialize (meta_use,
+ &sptr,
+ mds,
+ GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
+ if (NULL != meta)
+ GNUNET_CONTAINER_meta_data_destroy (meta);
+ if (ret == -1)
+ mds = 0;
+ else
+ mds = ret;
+ big = htonl (mds);
+ memcpy (&ser[slen], &big, sizeof (uint32_t));
+ e->len = slen + sizeof (uint32_t) + mds;
+ e->next = bld->head;
+ bld->head = e;
+ bld->count++;
+}
/**
* Given the start and end position of a block of
* data, return the end position of that data
- * after alignment to the GNUNET_FS_DBLOCK_SIZE.
+ * after alignment to the DBLOCK_SIZE.
*/
-static uint64_t
-do_align (uint64_t start_position,
- uint64_t end_position)
+static size_t
+do_align (size_t start_position,
+ size_t end_position)
{
- uint64_t align;
+ size_t align;
- align = (end_position / GNUNET_FS_DBLOCK_SIZE) * GNUNET_FS_DBLOCK_SIZE;
+ align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
if ((start_position < align) && (end_position > align))
return align + end_position - start_position;
return end_position;
* @param perm the permutation of the blocks (updated)
*/
static void
-block_align (uint64_t start,
+block_align (size_t start,
unsigned int count,
- const uint64_t *sizes,
+ const size_t *sizes,
unsigned int *perm)
{
unsigned int i;
unsigned int j;
unsigned int tmp;
unsigned int best;
- int64_t badness;
- uint64_t cpos;
- uint64_t cend;
- int64_t cbad;
+ ssize_t badness;
+ size_t cpos;
+ size_t cend;
+ ssize_t cbad;
unsigned int cval;
cpos = start;
{
cval = perm[j];
cend = cpos + sizes[cval];
- if (cpos % GNUNET_FS_DBLOCK_SIZE == 0)
+ if (cpos % DBLOCK_SIZE == 0)
{
/* prefer placing the largest blocks first */
- cbad = -(cend % GNUNET_FS_DBLOCK_SIZE);
+ cbad = -(cend % DBLOCK_SIZE);
}
else
{
- if (cpos / GNUNET_FS_DBLOCK_SIZE ==
- cend / GNUNET_FS_DBLOCK_SIZE)
+ if (cpos / DBLOCK_SIZE ==
+ cend / DBLOCK_SIZE)
{
/* Data fits into the same block! Prefer small left-overs! */
cbad =
- GNUNET_FS_DBLOCK_SIZE - cend % GNUNET_FS_DBLOCK_SIZE;
+ DBLOCK_SIZE - cend % DBLOCK_SIZE;
}
else
{
/* Would have to waste space to re-align, add big factor, this
case is a real loss (proportional to space wasted)! */
cbad =
- GNUNET_FS_DBLOCK_SIZE * (GNUNET_FS_DBLOCK_SIZE -
+ DBLOCK_SIZE * (DBLOCK_SIZE -
cpos %
- GNUNET_FS_DBLOCK_SIZE);
+ DBLOCK_SIZE);
}
}
if (cbad < badness)
badness = cbad;
}
}
+ GNUNET_assert (best != -1);
tmp = perm[i];
perm[i] = perm[best];
perm[best] = tmp;
/**
- * Create a directory. We allow packing more than one variable
- * size entry into one block (and an entry could also span more
- * than one block), but an entry that is smaller than a single
- * block will never cross the block boundary. This is done to
- * allow processing entries of a directory already even if the
- * download is still partial.<p>
- *
- * The first block begins with the directories MAGIC signature,
- * followed by the meta-data about the directory itself.<p>
- *
- * After that, the directory consists of block-aligned pairs
- * of URIs (0-terminated strings) and serialized meta-data.
+ * Finish building the directory. Frees the
+ * builder context and returns the directory
+ * in-memory.
*
- * @param data pointer set to the beginning of the directory
- * @param len set to number of bytes in data
- * @param count number of entries in uris and mds
- * @param uris URIs of the files in the directory
- * @param mds meta-data for the files (must match
- * respective values at same offset in in uris)
- * @param mdir meta-data for the directory
- * @return GNUNET_OK on success, GNUNET_SYSERR on error
+ * @param bld directory to finish
+ * @param rsize set to the number of bytes needed
+ * @param rdata set to the encoded directory
+ * @return GNUNET_OK on success
*/
int
-GNUNET_FS_directory_create (char **data,
- size_t *len,
- unsigned int count,
- const struct GNUNET_FS_Uri **uris,
- const struct GNUNET_CONTAINER_MetaData **mds,
- const struct GNUNET_CONTAINER_MetaData *mdir)
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+ size_t *rsize,
+ void **rdata)
{
+ char *data;
+ char *sptr;
+ size_t *sizes;
+ unsigned int *perm;
unsigned int i;
unsigned int j;
- uint64_t psize;
- uint64_t size;
- uint64_t pos;
- char **ucs;
- int ret;
- uint64_t *sizes;
- unsigned int *perm;
-
- for (i = 0; i < count; i++)
- {
- if (GNUNET_FS_uri_test_ksk (fis[i].uri))
- {
- GNUNET_break (0);
- return GNUNET_SYSERR; /* illegal in directory! */
- }
- }
- ucs = GNUNET_malloc (sizeof (char *) * count);
- size = 8 + sizeof (unsigned int);
- size += GNUNET_meta_data_get_serialized_size (meta, GNUNET_SERIALIZE_FULL);
- sizes = GNUNET_malloc (count * sizeof (unsigned long long));
- perm = GNUNET_malloc (count * sizeof (int));
- for (i = 0; i < count; i++)
+ struct BuilderEntry *pos;
+ struct BuilderEntry **bes;
+ size_t size;
+ size_t psize;
+ size_t off;
+ ssize_t ret;
+ uint32_t big;
+
+ size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof (uint32_t);
+ size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta);
+ sizes = NULL;
+ perm = NULL;
+ bes = NULL;
+ if (0 < bld->count)
{
- perm[i] = i;
- ucs[i] = GNUNET_FS_uri_to_string (fis[i].uri);
- GNUNET_assert (ucs[i] != NULL);
- psize =
- GNUNET_meta_data_get_serialized_size (fis[i].meta,
- GNUNET_SERIALIZE_FULL);
- if (psize == -1)
- {
- GNUNET_break (0);
- GNUNET_free (sizes);
- GNUNET_free (perm);
- while (i >= 0)
- GNUNET_free (ucs[i--]);
- GNUNET_free (ucs);
- return GNUNET_SYSERR;
- }
- sizes[i] = psize + sizeof (unsigned int) + strlen (ucs[i]) + 1;
+ sizes = GNUNET_malloc (bld->count * sizeof (size_t));
+ perm = GNUNET_malloc (bld->count * sizeof (unsigned int));
+ bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *));
+ pos = bld->head;
+ for (i = 0; i < bld->count; i++)
+ {
+ perm[i] = i;
+ bes[i] = pos;
+ sizes[i] = pos->len;
+ pos = pos->next;
+ }
+ block_align (size,
+ bld->count,
+ sizes,
+ perm);
+ /* compute final size with alignment */
+ for (i = 0; i < bld->count; i++)
+ {
+ psize = size;
+ size += sizes[perm[i]];
+ size = do_align (psize, size);
+ }
}
- /* permutate entries to minimize alignment cost */
- block_align (size, count, sizes, perm);
-
- /* compute final size with alignment */
- for (i = 0; i < count; i++)
+ *rsize = size;
+ data = GNUNET_malloc_large (size);
+ if (data == NULL)
{
- psize = size;
- size += sizes[perm[i]];
- size = do_align (psize, size);
+ GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+ "malloc");
+ *rsize = 0;
+ *rdata = NULL;
+ GNUNET_free_non_null (sizes);
+ GNUNET_free_non_null (perm);
+ GNUNET_free_non_null (bes);
+ return GNUNET_SYSERR;
}
- *len = size;
- *data = GNUNET_malloc (size);
- memset (*data, 0, size);
-
- pos = 8;
- memcpy (*data, GNUNET_DIRECTORY_MAGIC, 8);
-
- ret = GNUNET_CONTAINER_meta_data_serialize (meta,
- &(*data)[pos +
- sizeof (unsigned int)],
- size - pos - sizeof (unsigned int),
- GNUNET_SERIALIZE_FULL);
- GNUNET_assert (ret != GNUNET_SYSERR);
- ret = htonl (ret);
- memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
- pos += ntohl (ret) + sizeof (unsigned int);
-
- for (j = 0; j < count; j++)
+ *rdata = data;
+ memcpy (data, GNUNET_DIRECTORY_MAGIC, strlen (GNUNET_DIRECTORY_MAGIC));
+ off = strlen (GNUNET_DIRECTORY_MAGIC);
+
+ sptr = &data[off + sizeof (uint32_t)];
+ ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta,
+ &sptr,
+ size - off - sizeof (uint32_t),
+ GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ GNUNET_assert (ret != -1);
+ big = htonl (ret);
+ memcpy (&data[off], &big, sizeof (uint32_t));
+ off += sizeof (uint32_t) + ret;
+ for (j = 0; j < bld->count; j++)
{
i = perm[j];
- psize = pos;
- pos += sizes[i];
- pos = do_align (psize, pos);
- pos -= sizes[i]; /* go back to beginning */
- memcpy (&(*data)[pos], ucs[i], strlen (ucs[i]) + 1);
- pos += strlen (ucs[i]) + 1;
- GNUNET_free (ucs[i]);
- ret = GNUNET_CONTAINER_meta_data_serialize (mds[i],
- &(*data)[pos +
- sizeof (unsigned int)],
- size - pos -
- sizeof (unsigned int),
- GNUNET_SERIALIZE_FULL);
- GNUNET_assert (ret != GNUNET_SYSERR);
- ret = htonl (ret);
- memcpy (&(*data)[pos], &ret, sizeof (unsigned int));
- pos += ntohl (ret) + sizeof (unsigned int);
+ psize = off;
+ off += sizes[i];
+ off = do_align (psize, off);
+ memcpy (&data[off - sizes[i]],
+ &(bes[i])[1],
+ sizes[i]);
+ GNUNET_free (bes[i]);
}
- GNUNET_free (sizes);
- GNUNET_free (perm);
- GNUNET_free (ucs);
- GNUNET_assert (pos == size);
+ GNUNET_free_non_null (sizes);
+ GNUNET_free_non_null (perm);
+ GNUNET_free_non_null (bes);
+ GNUNET_assert (off == size);
+ GNUNET_CONTAINER_meta_data_destroy (bld->meta);
+ GNUNET_free (bld);
return GNUNET_OK;
}
-#endif
-
/* end of fs_directory.c */
projects / oweals / gnunet.git / blobdiff