2 This file is part of GNUnet.
3 (C) 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file util/container_meta_data.c
23 * @brief Storing of meta data
24 * @author Christian Grothoff
28 #include "gnunet_common.h"
29 #include "gnunet_container_lib.h"
30 #include "gnunet_strings_lib.h"
31 #include "gnunet_time_lib.h"
32 #include <extractor.h>
35 #define EXTRA_CHECKS ALLOW_EXTRA_CHECKS
39 EXTRACTOR_KeywordType type;
44 * Meta data to associate with a file, directory or namespace.
46 struct GNUNET_CONTAINER_MetaData
53 * Create a fresh struct CONTAINER_MetaData token.
55 struct GNUNET_CONTAINER_MetaData *
56 GNUNET_CONTAINER_meta_data_create ()
58 struct GNUNET_CONTAINER_MetaData *ret;
59 ret = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData));
69 GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md)
75 for (i = 0; i < md->itemCount; i++)
76 GNUNET_free (md->items[i].data);
77 GNUNET_array_grow (md->items, md->itemCount, 0);
82 * Add the current time as the publication date
86 GNUNET_CONTAINER_meta_data_add_publication_date (struct
87 GNUNET_CONTAINER_MetaData
91 struct GNUNET_TIME_Absolute t;
93 t = GNUNET_TIME_absolute_get ();
94 GNUNET_CONTAINER_meta_data_delete (md, EXTRACTOR_PUBLICATION_DATE, NULL);
95 dat = GNUNET_STRINGS_absolute_time_to_string (t);
96 GNUNET_CONTAINER_meta_data_insert (md, EXTRACTOR_PUBLICATION_DATE, dat);
102 * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists
105 GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
106 EXTRACTOR_KeywordType type,
112 GNUNET_assert (data != NULL);
113 for (idx = 0; idx < md->itemCount; idx++)
115 if ((md->items[idx].type == type) &&
116 (0 == strcmp (md->items[idx].data, data)))
117 return GNUNET_SYSERR;
120 GNUNET_array_grow (md->items, md->itemCount, md->itemCount + 1);
121 md->items[idx].type = type;
122 md->items[idx].data = p = GNUNET_strdup (data);
124 /* change OS native dir separators to unix '/' and others to '_' */
125 if (type == EXTRACTOR_FILENAME)
129 if (*p == DIR_SEPARATOR)
143 * @param md metadata to manipulate
144 * @param type type of the item to remove
145 * @param data specific value to remove, NULL to remove all
146 * entries of the given type
147 * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md
150 GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
151 EXTRACTOR_KeywordType type,
155 int ret = GNUNET_SYSERR;
156 for (idx = 0; idx < md->itemCount; idx++)
158 if ((md->items[idx].type == type) &&
159 ((data == NULL) || (0 == strcmp (md->items[idx].data, data))))
161 GNUNET_free (md->items[idx].data);
162 md->items[idx] = md->items[md->itemCount - 1];
163 GNUNET_array_grow (md->items, md->itemCount, md->itemCount - 1);
176 * Iterate over MD entries, excluding thumbnails.
178 * @param md metadata to inspect
179 * @param iter function to call on each entry
180 * @param iter_cls closure for iterator
181 * @return number of entries
184 GNUNET_CONTAINER_meta_data_get_contents (const struct
185 GNUNET_CONTAINER_MetaData *md,
186 GNUNET_CONTAINER_MetaDataProcessor
187 iter, void *iter_cls)
193 for (i = 0; i < md->itemCount; i++)
195 if (!EXTRACTOR_isBinaryType (md->items[i].type))
197 if ((iter != NULL) &&
198 (GNUNET_OK != iter (iter_cls,
199 md->items[i].type, md->items[i].data)))
200 return GNUNET_SYSERR;
205 return (int) (md->itemCount - sub);
209 * Iterate over MD entries
211 * @return number of entries
214 GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData
215 *md, EXTRACTOR_KeywordType type)
219 for (i = 0; i < md->itemCount; i++)
220 if (type == md->items[i].type)
221 return GNUNET_strdup (md->items[i].data);
226 * Iterate over MD entries
228 * @return number of entries
231 GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
232 GNUNET_CONTAINER_MetaData *md,
237 EXTRACTOR_KeywordType type;
243 type = va_arg (args, EXTRACTOR_KeywordType);
246 ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type);
255 * Get a thumbnail from the meta-data (if present).
257 * @param md metadata to get the thumbnail from
258 * @param thumb will be set to the thumbnail data. Must be
259 * freed by the caller!
260 * @return number of bytes in thumbnail, 0 if not available
263 GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
264 GNUNET_CONTAINER_MetaData * md,
265 unsigned char **thumb)
272 GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_THUMBNAIL_DATA);
275 if (strlen (encoded) == 0)
277 GNUNET_free (encoded);
278 return 0; /* invalid */
281 ret = EXTRACTOR_binaryDecode (encoded, thumb, &size);
282 GNUNET_free (encoded);
289 * Duplicate struct GNUNET_CONTAINER_MetaData.
291 * @param md what to duplicate
292 * @return duplicate meta-data container
294 struct GNUNET_CONTAINER_MetaData *
295 GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData
299 struct GNUNET_CONTAINER_MetaData *ret;
303 ret = GNUNET_CONTAINER_meta_data_create ();
304 for (i = 0; i < md->itemCount; i++)
305 GNUNET_CONTAINER_meta_data_insert (ret, md->items[i].type,
311 * Extract meta-data from a file.
313 * @return GNUNET_SYSERR on error, otherwise the number
314 * of meta-data items obtained
317 GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData
318 *md, const char *filename,
319 EXTRACTOR_ExtractorList *
322 EXTRACTOR_KeywordList *head;
323 EXTRACTOR_KeywordList *pos;
326 if (filename == NULL)
327 return GNUNET_SYSERR;
328 if (extractors == NULL)
330 head = EXTRACTOR_getKeywords (extractors, filename);
331 head = EXTRACTOR_removeDuplicateKeywords (head,
332 EXTRACTOR_DUPLICATES_REMOVE_UNKNOWN);
338 GNUNET_CONTAINER_meta_data_insert (md, pos->keywordType,
343 EXTRACTOR_freeKeywords (head);
349 tryCompression (char *data, unsigned int oldSize)
355 dlen = compressBound (oldSize);
357 dlen = oldSize + (oldSize / 100) + 20;
358 /* documentation says 100.1% oldSize + 12 bytes, but we
359 should be able to overshoot by more to be safe */
361 tmp = GNUNET_malloc (dlen);
362 if (Z_OK == compress2 ((Bytef *) tmp,
363 &dlen, (const Bytef *) data, oldSize, 9))
367 memcpy (data, tmp, dlen);
377 * Decompress input, return the decompressed data
378 * as output, set outputSize to the number of bytes
381 * @return NULL on error
384 decompress (const char *input,
385 unsigned int inputSize, unsigned int outputSize)
391 output = GNUNET_malloc (olen);
392 if (Z_OK == uncompress ((Bytef *) output,
393 &olen, (const Bytef *) input, inputSize))
399 GNUNET_free (output);
405 * Flag in 'version' that indicates compressed meta-data.
407 #define HEADER_COMPRESSED 0x80000000
410 * Bits in 'version' that give the version number.
412 #define HEADER_VERSION_MASK 0x7FFFFFFF
414 struct MetaDataHeader
417 * The version of the MD serialization.
418 * The highest bit is used to indicate
421 * Version 0 is the current version;
422 * Version is 1 for a NULL pointer.
423 * Other version numbers are not yet defined.
428 * How many MD entries are there?
433 * Size of the MD (decompressed)
438 * This is followed by 'entries' values of type 'unsigned int' that
439 * correspond to EXTRACTOR_KeywordTypes. After that, the meta-data
440 * keywords follow (0-terminated). The MD block always ends with
441 * 0-termination, padding with 0 until a multiple of 8 bytes.
447 * Serialize meta-data to target.
449 * @param md metadata to serialize
450 * @param target where to write the serialized metadata
451 * @param max maximum number of bytes available in target
452 * @param opt is it ok to just write SOME of the
453 * meta-data to match the size constraint,
454 * possibly discarding some data?
455 * @return number of bytes written on success,
456 * GNUNET_SYSERR on error (typically: not enough
460 GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
461 *md, char *target, size_t max,
463 GNUNET_CONTAINER_MetaDataSerializationOptions
466 struct MetaDataHeader *hdr;
473 if (max < sizeof (struct MetaDataHeader))
474 return GNUNET_SYSERR; /* far too small */
475 ic = md ? md->itemCount : 0;
479 size = sizeof (struct MetaDataHeader);
480 size += sizeof (uint32_t) * ic;
481 for (i = 0; i < ic; i++)
482 size += 1 + strlen (md->items[i].data);
483 while (size % 8 != 0)
485 hdr = GNUNET_malloc (size);
486 hdr->version = htonl (md == NULL ? 1 : 0);
487 hdr->entries = htonl (ic);
488 for (i = 0; i < ic; i++)
489 ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
490 pos = sizeof (struct MetaDataHeader);
491 pos += sizeof (unsigned int) * ic;
492 for (i = 0; i < ic; i++)
494 len = strlen (md->items[i].data) + 1;
495 memcpy (&((char *) hdr)[pos], md->items[i].data, len);
499 hdr->size = htonl (size);
500 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0)
502 pos = tryCompression ((char *) &hdr[1],
503 size - sizeof (struct MetaDataHeader));
507 pos = size - sizeof (struct MetaDataHeader);
509 if (pos < size - sizeof (struct MetaDataHeader))
511 hdr->version = htonl (HEADER_COMPRESSED);
512 size = pos + sizeof (struct MetaDataHeader);
519 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART) == 0)
521 return GNUNET_SYSERR; /* does not fit! */
523 /* partial serialization ok, try again with less meta-data */
525 ic = ic * 2 / 3; /* still far too big, make big reductions */
527 ic--; /* small steps, we're close */
529 GNUNET_assert (size <= max);
530 memcpy (target, hdr, size);
532 /* extra check: deserialize! */
535 struct GNUNET_CONTAINER_MetaData *mdx;
536 mdx = GNUNET_CONTAINER_meta_data_deserialize (target, size);
537 GNUNET_assert (NULL != mdx);
538 GNUNET_CONTAINER_meta_data_destroy (mdx);
545 * Estimate (!) the size of the meta-data in
546 * serialized form. The estimate MAY be higher
547 * than what is strictly needed.
549 * @param md metadata to inspect
550 * @param opt is it ok to just write SOME of the
551 * meta-data to match the size constraint,
552 * possibly discarding some data?
553 * @return number of bytes needed for serialization, -1 on error
556 GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
557 GNUNET_CONTAINER_MetaData *
560 GNUNET_CONTAINER_MetaDataSerializationOptions
563 struct MetaDataHeader *hdr;
570 ic = md ? md->itemCount : 0;
571 size = sizeof (struct MetaDataHeader);
572 size += sizeof (uint32_t) * ic;
573 for (i = 0; i < ic; i++)
574 size += 1 + strlen (md->items[i].data);
575 while (size % 8 != 0)
577 hdr = GNUNET_malloc (size);
578 hdr->version = htonl (md == NULL ? 1 : 0);
579 hdr->entries = htonl (ic);
580 for (i = 0; i < ic; i++)
581 ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
582 pos = sizeof (struct MetaDataHeader);
583 pos += sizeof (uint32_t) * ic;
584 for (i = 0; i < ic; i++)
586 len = strlen (md->items[i].data) + 1;
587 memcpy (&((char *) hdr)[pos], md->items[i].data, len);
590 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0)
593 tryCompression ((char *) &hdr[1],
594 size - sizeof (struct MetaDataHeader));
598 pos = size - sizeof (struct MetaDataHeader);
600 if (pos < size - sizeof (struct MetaDataHeader))
601 size = pos + sizeof (struct MetaDataHeader);
608 * Deserialize meta-data. Initializes md.
610 * @param input buffer with the serialized metadata
611 * @param size number of bytes available in input
612 * @return MD on success, NULL on error (i.e.
615 struct GNUNET_CONTAINER_MetaData *
616 GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
618 struct GNUNET_CONTAINER_MetaData *md;
619 const struct MetaDataHeader *hdr;
630 if (size < sizeof (struct MetaDataHeader))
632 hdr = (const struct MetaDataHeader *) input;
633 version = ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_VERSION_MASK;
635 return NULL; /* null pointer */
638 GNUNET_break_op (0); /* unsupported version */
641 ic = ntohl (MAKE_UNALIGNED (hdr->entries));
643 (ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_COMPRESSED) != 0;
647 ntohl (MAKE_UNALIGNED (hdr->size)) - sizeof (struct MetaDataHeader);
648 if (dataSize > 2 * 1042 * 1024)
651 return NULL; /* only 2 MB allowed [to make sure we don't blow
652 our memory limit because of a mal-formed
656 decompress ((const char *) &input[sizeof (struct MetaDataHeader)],
657 size - sizeof (struct MetaDataHeader), dataSize);
668 cdata = (const char *) &hdr[1];
669 dataSize = size - sizeof (struct MetaDataHeader);
670 if (size != ntohl (MAKE_UNALIGNED (hdr->size)))
677 if ((sizeof (uint32_t) * ic + ic) > dataSize)
682 if ((ic > 0) && (cdata[dataSize - 1] != '\0'))
688 md = GNUNET_CONTAINER_meta_data_create ();
689 GNUNET_array_grow (md->items, md->itemCount, ic);
691 pos = sizeof (uint32_t) * ic;
692 while ((pos < dataSize) && (i < ic))
694 len = strlen (&cdata[pos]) + 1;
695 md->items[i].type = (EXTRACTOR_KeywordType)
696 ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i]));
697 md->items[i].data = GNUNET_strdup (&cdata[pos]);
703 GNUNET_CONTAINER_meta_data_destroy (md);
706 GNUNET_free_non_null (data);
709 GNUNET_free_non_null (data);
710 return NULL; /* size too small */
714 * Test if two MDs are equal.
716 * @param md1 first value to check
717 * @param md2 other value to check
718 * @return GNUNET_YES if they are equal
721 GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData
723 const struct GNUNET_CONTAINER_MetaData
730 if (md1->itemCount != md2->itemCount)
732 for (i = 0; i < md1->itemCount; i++)
735 for (j = 0; j < md2->itemCount; j++)
736 if ((md1->items[i].type == md2->items[j].type) &&
737 (0 == strcmp (md1->items[i].data, md2->items[j].data)))
742 if (found == GNUNET_NO)
749 /* end of container_meta_data.c */