+/**
+ * Add a keyword as non-mandatory (with ' '-prefix) to the
+ * given keyword list at offset 'index'. The array is
+ * guaranteed to be long enough.
+ *
+ * @param s keyword to add
+ * @param array array to add the keyword to
+ * @param index offset where to add the keyword
+ */
+static void
+insert_non_mandatory_keyword (const char *s, char **array, int index)
+{
+ char *nkword;
+ GNUNET_asprintf (&nkword, " %s", /* space to mark as 'non mandatory' */ s);
+ array[index] = nkword;
+}
+
+
+/**
+ * Test if the given keyword 's' is already present in the
+ * given array, ignoring the '+'-mandatory prefix in the array.
+ *
+ * @param s keyword to test
+ * @param array keywords to test against, with ' ' or '+' prefix to ignore
+ * @param array_length length of the array
+ * @return GNUNET_YES if the keyword exists, GNUNET_NO if not
+ */
+static int
+find_duplicate (const char *s, const char **array, int array_length)
+{
+ int j;
+
+ for (j = array_length - 1; j >= 0; j--)
+ if (0 == strcmp (&array[j][1], s))
+ return GNUNET_YES;
+ return GNUNET_NO;
+}
+
+
+/**
+ * FIXME: comment
+ */
+static char *
+normalize_metadata (enum EXTRACTOR_MetaFormat format, const char *data,
+ size_t data_len)
+{
+ uint8_t *free_str = NULL;
+ uint8_t *str_to_normalize = (uint8_t *) data;
+ uint8_t *normalized;
+ size_t r_len;
+ if (str_to_normalize == NULL)
+ return NULL;
+ /* Don't trust libextractor */
+ if (format == EXTRACTOR_METAFORMAT_UTF8)
+ {
+ free_str = (uint8_t *) u8_check ((const uint8_t *) data, data_len);
+ if (free_str == NULL)
+ free_str = NULL;
+ else
+ format = EXTRACTOR_METAFORMAT_C_STRING;
+ }
+ if (format == EXTRACTOR_METAFORMAT_C_STRING)
+ {
+ free_str = u8_strconv_from_encoding (data, locale_charset (), iconveh_escape_sequence);
+ if (free_str == NULL)
+ return NULL;
+ }
+
+ normalized = u8_tolower (str_to_normalize, strlen ((char *) str_to_normalize), NULL, UNINORM_NFD, NULL, &r_len);
+ /* free_str is allocated by libunistring internally, use free() */
+ if (free_str != NULL)
+ free (free_str);
+ if (normalized != NULL)
+ {
+ /* u8_tolower allocates a non-NULL-terminated string! */
+ free_str = GNUNET_malloc (r_len + 1);
+ memcpy (free_str, normalized, r_len);
+ free_str[r_len] = '\0';
+ free (normalized);
+ normalized = free_str;
+ }
+ return (char *) normalized;
+}
+
+/**
+ * Counts the number of UTF-8 characters (not bytes) in the string,
+ * returns that count.
+ */
+static size_t
+u8_strcount (const uint8_t *s)
+{
+ size_t count;
+ ucs4_t c;
+ GNUNET_assert (s != NULL);
+ if (s[0] == 0)
+ return 0;
+ for (count = 0; s != NULL; count++)
+ s = u8_next (&c, s);
+ return count - 1;
+}
+
+
+/**
+ * Break the filename up by matching [], () and {} pairs to make
+ * keywords. In case of nesting parentheses only the inner pair counts.
+ * You can't escape parentheses to scan something like "[blah\{foo]" to
+ * make a "blah{foo" keyword, this function is only a heuristic!
+ *
+ * @param s string to break down.
+ * @param array array to fill with enclosed tokens. If NULL, then tokens
+ * are only counted.
+ * @param index index at which to start filling the array (entries prior
+ * to it are used to check for duplicates). ignored if array == NULL.
+ * @return number of tokens counted (including duplicates), or number of
+ * tokens extracted (excluding duplicates). 0 if there are no
+ * matching parens in the string (when counting), or when all tokens
+ * were duplicates (when extracting).
+ */
+static int
+get_keywords_from_parens (const char *s, char **array, int index)
+{
+ int count = 0;
+ char *open_paren;
+ char *close_paren;
+ char *ss;
+ char tmp;
+
+ if (NULL == s)
+ return 0;
+ ss = GNUNET_strdup (s);
+ open_paren = ss - 1;
+ while (NULL != (open_paren = strpbrk (open_paren + 1, "[{(")))
+ {
+ int match = 0;
+
+ close_paren = strpbrk (open_paren + 1, "]})");
+ if (NULL == close_paren)
+ continue;
+ switch (open_paren[0])
+ {
+ case '[':
+ if (']' == close_paren[0])
+ match = 1;
+ break;
+ case '{':
+ if ('}' == close_paren[0])
+ match = 1;
+ break;
+ case '(':
+ if (')' == close_paren[0])
+ match = 1;
+ break;
+ default:
+ break;
+ }
+ if (match && (close_paren - open_paren > 1))
+ {
+ tmp = close_paren[0];
+ close_paren[0] = '\0';
+ /* Keywords must be at least 3 characters long */
+ if (u8_strcount ((const uint8_t *) &open_paren[1]) <= 2)
+ {
+ close_paren[0] = tmp;
+ continue;
+ }
+ if (NULL != array)
+ {
+ char *normalized;
+ if (GNUNET_NO == find_duplicate ((const char *) &open_paren[1],
+ (const char **) array, index + count))
+ {
+ insert_non_mandatory_keyword ((const char *) &open_paren[1], array,
+ index + count);
+ count++;
+ }
+ normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8,
+ &open_paren[1], close_paren - &open_paren[1]);
+ if (normalized != NULL)
+ {
+ if (GNUNET_NO == find_duplicate ((const char *) normalized,
+ (const char **) array, index + count))
+ {
+ insert_non_mandatory_keyword ((const char *) normalized, array,
+ index + count);
+ count++;
+ }
+ GNUNET_free (normalized);
+ }
+ }
+ else
+ count++;
+ close_paren[0] = tmp;
+ }
+ }
+ GNUNET_free (ss);
+ return count;
+}
+
+
+/**
+ * Where to break up keywords
+ */
+#define TOKENS "_. /-!?#&+@\"\'\\;:,"
+
+/**
+ * Break the filename up by TOKENS to make
+ * keywords.
+ *
+ * @param s string to break down.
+ * @param array array to fill with tokens. If NULL, then tokens are only
+ * counted.
+ * @param index index at which to start filling the array (entries prior
+ * to it are used to check for duplicates). ignored if array == NULL.
+ * @return number of tokens (>1) counted (including duplicates), or number of
+ * tokens extracted (excluding duplicates). 0 if there are no
+ * separators in the string (when counting), or when all tokens were
+ * duplicates (when extracting).
+ */
+static int
+get_keywords_from_tokens (const char *s, char **array, int index)
+{
+ char *p;
+ char *ss;
+ int seps = 0;
+
+ ss = GNUNET_strdup (s);
+ for (p = strtok (ss, TOKENS); p != NULL; p = strtok (NULL, TOKENS))
+ {
+ /* Keywords must be at least 3 characters long */
+ if (u8_strcount ((const uint8_t *) p) <= 2)
+ continue;
+ if (NULL != array)
+ {
+ char *normalized;
+ if (GNUNET_NO == find_duplicate (p, (const char **) array, index + seps))
+ {
+ insert_non_mandatory_keyword (p, array,
+ index + seps);
+ seps++;
+ }
+ normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8,
+ p, strlen (p));
+ if (normalized != NULL)
+ {
+ if (GNUNET_NO == find_duplicate ((const char *) normalized,
+ (const char **) array, index + seps))
+ {
+ insert_non_mandatory_keyword ((const char *) normalized, array,
+ index + seps);
+ seps++;
+ }
+ GNUNET_free (normalized);
+ }
+ }
+ else
+ seps++;
+ }
+ GNUNET_free (ss);
+ return seps;
+}
+#undef TOKENS
+
projects / oweals / gnunet.git / blobdiff