search syncing

diff --git a/ChangeLog b/ChangeLog
index 8f14863eb4db97ff06dbc0ba7c064d67935868c8..7cb118fbd425bed5e2e670ec8d6ad101af37e724 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,6 @@
 Sun May  2 13:49:10 CEST 2010
        Fixed problem with platform-dependence of format for IP addresses
 Sun May  2 13:49:10 CEST 2010
        Fixed problem with platform-dependence of format for IP addresses

-       in HELLOs for TCP and UDP transport. 
+       in HELLOs for TCP and UDP transport.

        Reduced address length field in HELLOs to 16 bit (was 32 bit).
        These changes break transport compatibility.
 
        Reduced address length field in HELLOs to 16 bit (was 32 bit).
        These changes break transport compatibility.
 

diff --git a/TODO b/TODO
index 5d9a03b75d720bae112590bc6c9f6ed0a5ce252a..df1836f06587c9312909973d9fe9fd90c9c249c3 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,6 +1,8 @@
 0.9.0pre1:
 * FS: [CG]
 0.9.0pre1:
 * FS: [CG]

-  - serialization and deserialization code (search, download)
+  - deserialization code (download)
+  - serialization code (download)
+  - linking of downloads to searches 

   - generate SUSPEND events (publish, unindex, search, download)
   - actually call 'sync' functions (publish, unindex, search, download)
   - persistence testing (publish, unindex)
   - generate SUSPEND events (publish, unindex, search, download)
   - actually call 'sync' functions (publish, unindex, search, download)
   - persistence testing (publish, unindex)


@@ -71,6 +73,9 @@
 * FS: [CG]
   - datastore reservation (publishing)
   - location URIs (publish, search, download)
 * FS: [CG]
   - datastore reservation (publishing)
   - location URIs (publish, search, download)

+  - utilize in-line files in meta data always (including in search results or
+    when download is triggered manually and for probes); currently the data is
+    only used when users do a general 'recursive' download

   - non-anonymous FS service (needs DHT)
     + DHT integration for search
     + CS-DHT-functions (DHT-put of LOC)
   - non-anonymous FS service (needs DHT)
     + DHT integration for search
     + CS-DHT-functions (DHT-put of LOC)

diff --git a/src/fs/fs.c b/src/fs/fs.c
index 5b8dc3e90cffb03096399f71a45be4a676cc7e60..6556fee18857f67f042af849fe31a12eea59a865 100644 (file)
--- a/src/fs/fs.c
+++ b/src/fs/fs.c
@@ -211,8 +211,6 @@ GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh)
 }
 
 
 }
 
 

-
-

 /**
  * Closure for "data_reader_file".
  */
 /**
  * Closure for "data_reader_file".
  */


@@ -1251,12 +1249,56 @@ GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc)
  * publishing structs should already call "sync" internally,
  * so this function is likely not useful for clients.
  * 
  * publishing structs should already call "sync" internally,
  * so this function is likely not useful for clients.
  * 

- * @param sc the struct to sync
+ * @param key key for the search result
+ * @param sr the struct to sync

  */
 void
  */
 void

-GNUNET_FS_search_result_sync_ (struct SearchResult *sr)
+GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key,
+                              struct SearchResult *sr)

 {
 {

-  /* FIXME */
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *uris;
+
+  GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_chk (sr->uri)) ||
+                 (GNUNET_YES == GNUNET_FS_uri_test_loc (sr->uri)) );
+  uris = NULL;
+  if (NULL == sr->serialization)
+    sr->serialization = make_serialization_file_name (sr->sc->h,
+                                                     "search-results");
+  if (NULL == sr->serialization)
+    return;
+  wh = get_write_handle (sr->sc->h, "search-results", sr->serialization);
+  uris = GNUNET_FS_uri_to_string (sr->uri);
+  if ( (GNUNET_OK !=
+       GNUNET_BIO_write_string (wh, uris)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write_meta_data (wh, sr->meta)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write (wh, key, sizeof (GNUNET_HashCode))) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write_int32 (wh, sr->mandatory_missing)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write_int32 (wh, sr->optional_support)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write_int32 (wh, sr->availability_success)) ||
+       (GNUNET_OK !=
+       GNUNET_BIO_write_int32 (wh, sr->availability_trials)) )
+    goto cleanup;   
+  if (GNUNET_OK !=
+      GNUNET_BIO_write_close (wh))
+    {
+      wh = NULL;
+      goto cleanup;
+    }
+  GNUNET_free_non_null (uris);
+  return;
+ cleanup:
+  GNUNET_free_non_null (uris);
+  if (wh != NULL)
+    (void)  GNUNET_BIO_write_close (wh);
+  GNUNET_FS_remove_sync_file_ (sr->sc->h, "search-results", sr->serialization);
+  GNUNET_free (sr->serialization);
+  sr->serialization = NULL;

 }
 
 
 }
 
 


@@ -1272,6 +1314,9 @@ void
 GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc)
 {  
   struct GNUNET_BIO_WriteHandle *wh;
 GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc)
 {  
   struct GNUNET_BIO_WriteHandle *wh;

+  struct GNUNET_FS_SearchContext *scc;
+  char *uris;
+  char in_pause;

 
   if (NULL == sc->serialization)
     sc->serialization = make_serialization_file_name (sc->h,
 
   if (NULL == sc->serialization)
     sc->serialization = make_serialization_file_name (sc->h,


@@ -1279,38 +1324,50 @@ GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc)
   if (NULL == sc->serialization)
     return;
   wh = get_write_handle (sc->h, "search", sc->serialization);
   if (NULL == sc->serialization)
     return;
   wh = get_write_handle (sc->h, "search", sc->serialization);

-#if 0
+  GNUNET_assert ( (GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) ||
+                 (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri)) );
+  uris = GNUNET_FS_uri_to_string (sc->uri);
+  in_pause = (sc->task != GNUNET_SCHEDULER_NO_TASK) ? 'r' : '\0';

   if ( (GNUNET_OK !=
   if ( (GNUNET_OK !=

-       GNUNET_BIO_write_string (wh, pc->nid)) ||
+       GNUNET_BIO_write_string (wh, uris)) ||

        (GNUNET_OK !=
        (GNUNET_OK !=

-       GNUNET_BIO_write_string (wh, pc->nuid)) ||
+       GNUNET_BIO_write_int64 (wh, sc->start_time.value)) ||

        (GNUNET_OK !=
        (GNUNET_OK !=

-       GNUNET_BIO_write_int32 (wh, pc->options)) ||
+       GNUNET_BIO_write_string (wh, sc->emsg)) ||

        (GNUNET_OK !=
        (GNUNET_OK !=

-       GNUNET_BIO_write_int32 (wh, pc->all_done)) ||
+       GNUNET_BIO_write_int32 (wh, (uint32_t) sc->options)) ||

        (GNUNET_OK !=
        (GNUNET_OK !=

-       GNUNET_BIO_write_string (wh, pc->fi->serialization)) ||
-       (GNUNET_OK !=
-       GNUNET_BIO_write_string (wh, (pc->fi_pos == NULL) ? NULL : pc->fi_pos->serialization)) ||
+       GNUNET_BIO_write (wh, &in_pause, sizeof (in_pause))) ||

        (GNUNET_OK !=
        (GNUNET_OK !=

-       GNUNET_BIO_write_string (wh, (pc->namespace == NULL) ? NULL : pc->namespace->name)) )
-   {
-     (void) GNUNET_BIO_write_close (wh);
-     GNUNET_FS_remove_sync_file_ (pc->h, "publish", pc->serialization);
-     GNUNET_free (pc->serialization);
-     pc->serialization = NULL;
-     return;
-   }
-#endif
-  /* FIXME: do search-specific serialization here! */
+       GNUNET_BIO_write_int32 (wh, sc->anonymity)) )
+    goto cleanup;          
+  GNUNET_free (uris);
+  uris = NULL;
+  scc = sc->child_head;
+  while (NULL != scc)
+    {
+      if (scc->serialization == NULL)
+       break;
+      if (GNUNET_OK !=
+         GNUNET_BIO_write_string (wh, scc->serialization))
+       goto cleanup;
+      scc = scc->next;
+    }
+  GNUNET_BIO_write_string (wh, NULL);

   if (GNUNET_OK !=
       GNUNET_BIO_write_close (wh))
     {
   if (GNUNET_OK !=
       GNUNET_BIO_write_close (wh))
     {

-      GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization);
-      GNUNET_free (sc->serialization);
-      sc->serialization = NULL;
-      return;     
-    }  
+      wh = NULL;
+      goto cleanup;
+    }
+  return;
+ cleanup:
+  if (wh != NULL)
+    (void) GNUNET_BIO_write_close (wh);
+  GNUNET_free_non_null (uris);
+  GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization);
+  GNUNET_free (sc->serialization);
+  sc->serialization = NULL;

 }
 
 
 }
 
 


@@ -1332,8 +1389,6 @@ deserialize_publish (struct GNUNET_FS_Handle *h)
 }
 
 
 }
 
 

-
-

 /**
  * Function called with a filename of serialized unindexing operation
  * to deserialize.
 /**
  * Function called with a filename of serialized unindexing operation
  * to deserialize.


@@ -1609,6 +1664,41 @@ free_result (void *cls,
 }
 
 
 }
 
 

+/**
+ * Free memory allocated by the search context and its children
+ *
+ * @param sc search context to free
+ */
+static void
+free_search_context (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_FS_SearchContext *scc;
+
+  while (NULL != (scc = sc->child_head))
+    {
+      GNUNET_CONTAINER_DLL_remove (sc->child_head,
+                                  sc->child_tail,
+                                  scc);      
+      free_search_context (scc);
+    }
+  GNUNET_free_non_null (sc->emsg);
+  if (sc->serialization != NULL)
+    GNUNET_FS_remove_sync_file_ (sc->h, "search", sc->serialization);
+  /* FIXME: remove 'pbuf' directory with search results as well! */
+  GNUNET_free_non_null (sc->serialization);
+  if (sc->uri != NULL)
+    GNUNET_FS_uri_destroy (sc->uri);
+  if (sc->master_result_map != NULL)
+    {
+      GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                            &free_result,
+                                            sc);
+      GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
+    }
+  GNUNET_free (sc);
+}
+
+

 /**
  * Deserialize a search. 
  *
 /**
  * Deserialize a search. 
  *


@@ -1624,6 +1714,8 @@ deserialize_search (struct GNUNET_FS_Handle *h,
                    const char *serialization)
 {
   struct GNUNET_FS_SearchContext *sc;
                    const char *serialization)
 {
   struct GNUNET_FS_SearchContext *sc;

+  struct GNUNET_FS_SearchContext *scc;
+  struct GNUNET_BIO_ReadHandle *rhc;

   char pbuf[32];
   struct GNUNET_FS_ProgressInfo pi;
   char *emsg;
   char pbuf[32];
   struct GNUNET_FS_ProgressInfo pi;
   char *emsg;


@@ -1634,7 +1726,6 @@ deserialize_search (struct GNUNET_FS_Handle *h,
   char in_pause;
 
   uris = NULL;
   char in_pause;
 
   uris = NULL;

-  child_ser = NULL;

   emsg = NULL;
   sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext));
   sc->parent = parent;
   emsg = NULL;
   sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext));
   sc->parent = parent;


@@ -1647,8 +1738,6 @@ deserialize_search (struct GNUNET_FS_Handle *h,
         (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri)) ) ||
        (GNUNET_OK !=
        GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) ||
         (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri)) ) ||
        (GNUNET_OK !=
        GNUNET_BIO_read_int64 (rh, &sc->start_time.value)) ||

-       (GNUNET_OK !=
-       GNUNET_BIO_read_string (rh, "child-serialization", &child_ser, 32)) ||

        (GNUNET_OK !=
        GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) ||
        (GNUNET_OK !=
        (GNUNET_OK !=
        GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10*1024)) ||
        (GNUNET_OK !=


@@ -1679,9 +1768,33 @@ deserialize_search (struct GNUNET_FS_Handle *h,
          GNUNET_FS_search_start_searching_ (sc))
        goto cleanup;
     }
          GNUNET_FS_search_start_searching_ (sc))
        goto cleanup;
     }

-  if (child_ser != NULL)
+  while (1)

     {
     {

-      /* FIXME: deserialize child-search! */
+      if ( (GNUNET_OK !=
+           GNUNET_BIO_read_string (rh, "child-serialization", &child_ser, 32)))
+       goto cleanup;
+      if (child_ser == NULL)
+       break;    
+      rhc = get_read_handle (h, "search-children", child_ser);
+      if (rhc != NULL)
+       {
+         scc = deserialize_search (h, rhc, sc, child_ser);
+         if (scc != NULL)          
+           GNUNET_CONTAINER_DLL_insert (sc->child_head,
+                                        sc->child_tail,
+                                        scc);      
+         emsg = NULL;
+         if (GNUNET_OK !=
+             GNUNET_BIO_read_close (rhc, &emsg))
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         _("Failed to resume sub-search `%s': %s\n"),
+                         child_ser,
+                         emsg);
+             GNUNET_free (emsg);
+           }
+       }    
+      GNUNET_free (child_ser);  

     }
   if (parent != NULL)
     GNUNET_CONTAINER_DLL_insert (parent->child_head,
     }
   if (parent != NULL)
     GNUNET_CONTAINER_DLL_insert (parent->child_head,


@@ -1692,28 +1805,25 @@ deserialize_search (struct GNUNET_FS_Handle *h,
   pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES;
   sc->client_info = GNUNET_FS_search_make_status_ (&pi,
                                                   sc);
   pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES;
   sc->client_info = GNUNET_FS_search_make_status_ (&pi,
                                                   sc);

+  scc = sc->child_head;
+  while (NULL != scc)
+    {
+      pi.status = GNUNET_FS_STATUS_SEARCH_RESUME;
+      pi.value.search.specifics.resume.message = scc->emsg;
+      pi.value.search.specifics.resume.is_paused = ('\0' == in_pause) ? GNUNET_NO : GNUNET_YES;
+      scc->client_info = GNUNET_FS_search_make_status_ (&pi,
+                                                       scc);
+
+      scc = scc->next;
+    }

   GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                         &signal_result_resume,
                                         sc);
   GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                         &signal_result_resume,
                                         sc);

+  GNUNET_free (uris);

   return sc;
  cleanup:
   return sc;
  cleanup:

-  GNUNET_free_non_null (child_ser);
-  GNUNET_free_non_null (sc->emsg);

   GNUNET_free_non_null (emsg);
   GNUNET_free_non_null (emsg);

-  if (sc->serialization != NULL)
-    GNUNET_FS_remove_sync_file_ (h, "search", sc->serialization);
-  /* FIXME: remove 'pbuf' directory with search results as well! */
-  GNUNET_free_non_null (sc->serialization);
-  if (sc->uri != NULL)
-    GNUNET_FS_uri_destroy (sc->uri);
-  if (sc->master_result_map != NULL)
-    {
-      GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
-                                            &free_result,
-                                            sc);
-      GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
-    }
-  GNUNET_free (sc);
+  free_search_context (sc);

   GNUNET_free_non_null (uris);
   return NULL;
 }
   GNUNET_free_non_null (uris);
   return NULL;
 }

diff --git a/src/fs/fs.h b/src/fs/fs.h
index 33ee8dde003a816ac9f1272e5751774121c72b8f..338b38fc761a6702e5b4054f1c5193982abeebee 100644 (file)
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -880,10 +880,12 @@ GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc);
  * publishing structs should already call "sync" internally,
  * so this function is likely not useful for clients.
  * 
  * publishing structs should already call "sync" internally,
  * so this function is likely not useful for clients.
  * 

- * @param sc the struct to sync
+ * @param key key for the search result
+ * @param sr the struct to sync

  */
 void
  */
 void

-GNUNET_FS_search_result_sync_ (struct SearchResult *sr);
+GNUNET_FS_search_result_sync_ (const GNUNET_HashCode *key,
+                              struct SearchResult *sr);

 
 
 /**
 
 
 /**