From: Christian Grothoff Date: Sun, 25 Oct 2009 20:12:05 +0000 (+0000) Subject: fixing indexing and unindexing issues X-Git-Tag: initial-import-from-subversion-38251~23247 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=c22ef5d895b1dba3555f5e012770265a9372dc8e;p=oweals%2Fgnunet.git fixing indexing and unindexing issues --- diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 38db1bc58..59361b03e 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -118,6 +118,7 @@ TESTS = \ test_fs_download \ test_fs_search \ test_fs_start_stop \ + test_fs_unindex \ test_fs_uri # $(check_PROGRAMS) diff --git a/src/fs/fs.h b/src/fs/fs.h index 2cac304a3..acfc8b1c6 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -367,6 +367,18 @@ struct GNUNET_FS_FileInformation */ int do_index; + /** + * Is "file_id" already valid? Set to GNUNET_YES + * once the hash has been calculated. + */ + int have_hash; + + /** + * Has the service confirmed our INDEX_START request? + * GNUNET_YES if this step has been completed. + */ + int index_start_confirmed; + } file; /** diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index f4f4fb567..330ddc2fc 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -202,6 +202,7 @@ GNUNET_FS_file_information_create_from_file (void *client_info, { struct FileInfo *fi; struct stat sbuf; + struct GNUNET_FS_FileInformation *ret; if (0 != STAT (filename, &sbuf)) { @@ -212,16 +213,18 @@ GNUNET_FS_file_information_create_from_file (void *client_info, } fi = GNUNET_malloc (sizeof(struct FileInfo)); fi->filename = GNUNET_strdup (filename); - return GNUNET_FS_file_information_create_from_reader (client_info, - sbuf.st_size, - &data_reader_file, - fi, - keywords, - meta, - do_index, - anonymity, - priority, - expirationTime); + ret = GNUNET_FS_file_information_create_from_reader (client_info, + sbuf.st_size, + &data_reader_file, + fi, + keywords, + meta, + do_index, + anonymity, + priority, + expirationTime); + ret->data.file.filename = GNUNET_strdup (filename); + return ret; } diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 34a13f55c..a64f0d15e 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -482,13 +482,6 @@ block_proc (void *cls, struct PutContCtx * dpc_cls; struct OnDemandBlock odb; -#if DEBUG_PUBLISH - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Publishing block `%s' for offset %llu with size %u\n", - GNUNET_h2s (query), - (unsigned long long) offset, - (unsigned int) block_size); -#endif p = sc->fi_pos; if (NULL == sc->dsh) { @@ -510,10 +503,17 @@ block_proc (void *cls, dpc_cls->cont_cls = sc; dpc_cls->sc = sc; dpc_cls->p = p; - if ( (p->is_directory) && - (p->data.file.do_index) && + if ( (! p->is_directory) && + (GNUNET_YES == p->data.file.do_index) && (type == GNUNET_DATASTORE_BLOCKTYPE_DBLOCK) ) { +#if DEBUG_PUBLISH + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Indexing block `%s' for offset %llu with index size %u\n", + GNUNET_h2s (query), + (unsigned long long) offset, + sizeof (struct OnDemandBlock)); +#endif odb.offset = offset; odb.file_id = p->data.file.file_id; GNUNET_DATASTORE_put (sc->dsh, @@ -530,6 +530,13 @@ block_proc (void *cls, dpc_cls); return; } +#if DEBUG_PUBLISH + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Publishing block `%s' for offset %llu with size %u\n", + GNUNET_h2s (query), + (unsigned long long) offset, + (unsigned int) block_size); +#endif GNUNET_DATASTORE_put (sc->dsh, sc->rid, query, @@ -702,6 +709,7 @@ process_index_start_response (void *cls, publish_content (sc); return; } + p->data.file.index_start_confirmed = GNUNET_YES; /* success! continue with indexing */ publish_content (sc); } @@ -738,6 +746,11 @@ hash_for_index_cb (void *cls, publish_content (sc); return; } + if (GNUNET_YES == p->data.file.index_start_confirmed) + { + publish_content (sc); + return; + } slen = strlen (p->data.file.filename) + 1; if (slen > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage)) { @@ -749,6 +762,12 @@ hash_for_index_cb (void *cls, publish_content (sc); return; } +#if DEBUG_PUBLISH + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Hash of indexed file `%s' is `%s'\n", + p->data.file.filename, + GNUNET_h2s (res)); +#endif client = GNUNET_CLIENT_connect (sc->h->sched, "fs", sc->h->cfg); @@ -763,6 +782,7 @@ hash_for_index_cb (void *cls, return; } p->data.file.file_id = *res; + p->data.file.have_hash = GNUNET_YES; ism = GNUNET_malloc (sizeof(struct IndexStartMessage) + slen); ism->header.size = htons(sizeof(struct IndexStartMessage) + @@ -776,6 +796,13 @@ hash_for_index_cb (void *cls, ism->device = htonl (dev); ism->inode = GNUNET_htonll(ino); } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to get file identifiers for `%s'\n"), + p->data.file.filename); + } + ism->file_id = *res; memcpy (&ism[1], p->data.file.filename, slen); @@ -877,13 +904,17 @@ do_upload (void *cls, publish_content (sc); return; } - GNUNET_CRYPTO_hash_file (sc->h->sched, - GNUNET_SCHEDULER_PRIORITY_IDLE, - GNUNET_NO, - p->data.file.filename, - HASHING_BLOCKSIZE, - &hash_for_index_cb, - sc); + if (p->data.file.have_hash) + hash_for_index_cb (sc, + &p->data.file.file_id); + else + GNUNET_CRYPTO_hash_file (sc->h->sched, + GNUNET_SCHEDULER_PRIORITY_IDLE, + GNUNET_NO, + p->data.file.filename, + HASHING_BLOCKSIZE, + &hash_for_index_cb, + sc); return; } publish_content (sc); diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c index 10faf4cf9..3d39cb932 100644 --- a/src/fs/fs_unindex.c +++ b/src/fs/fs_unindex.c @@ -20,12 +20,9 @@ /** * @file fs/fs_unindex.c - * @author Krista Bennett + * @author Krista Grothoff * @author Christian Grothoff * @brief Unindex file. - * - * TODO: - * - code cleanup (share more with upload.c) */ #include "platform.h" #include "gnunet_constants.h" diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index dc97b2cb8..5148c4675 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -990,8 +990,14 @@ hash_for_index_val (void *cls, sizeof(GNUNET_HashCode))) ) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - _("Hash mismatch trying to index file `%s'\n"), - ii->filename); + _("Hash mismatch trying to index file `%s' which has hash `%s'\n"), + ii->filename, + GNUNET_h2s (res)); +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Wanted `%s'\n", + GNUNET_h2s (&ii->file_id)); +#endif GNUNET_SERVER_transmit_context_append (ii->tc, NULL, 0, GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED); @@ -1058,6 +1064,14 @@ handle_index_start (void *cls, signal_index_ok (ii); return; } +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n", + (unsigned long long) ino, + (unsigned long long) myino, + (unsigned int) dev, + (unsigned int) mydev); +#endif /* slow validation, need to hash full file (again) */ GNUNET_CRYPTO_hash_file (sched, GNUNET_SCHEDULER_PRIORITY_IDLE, @@ -1165,7 +1179,13 @@ handle_unindex (void *cls, } pos = next; } - if (GNUNET_YES == found) +#if DEBUG_FS + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Client requested unindexing of file `%s': %s\n", + GNUNET_h2s (&um->file_id), + found ? "found" : "not found"); +#endif + if (GNUNET_YES == found) write_index_list (); tc = GNUNET_SERVER_transmit_context_create (client); GNUNET_SERVER_transmit_context_append (tc, diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c index 1d1813c56..9b85e1ca1 100644 --- a/src/fs/test_fs_download.c +++ b/src/fs/test_fs_download.c @@ -116,7 +116,7 @@ progress_cb (void *cls, case GNUNET_FS_STATUS_PUBLISH_COMPLETED: printf ("Publishing complete, %llu kbps.\n", (unsigned long long) (FILESIZE * 1000 / (1+GNUNET_TIME_absolute_get_duration (start).value) / 1024)); - fn = GNUNET_DISK_mktemp ("gnunet-download-test-dstXXXXXX"); + fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); start = GNUNET_TIME_absolute_get (); download = GNUNET_FS_download_start (fs, event->value.publish.specifics.completed.chk_uri, diff --git a/src/fs/test_fs_unindex.c b/src/fs/test_fs_unindex.c index 0b7173115..3dcf7b7f8 100644 --- a/src/fs/test_fs_unindex.c +++ b/src/fs/test_fs_unindex.c @@ -29,7 +29,7 @@ #include "gnunet_arm_service.h" #include "gnunet_fs_service.h" -#define VERBOSE GNUNET_NO +#define VERBOSE GNUNET_YES #define START_ARM GNUNET_YES @@ -248,7 +248,7 @@ run (void *cls, sched = s; setup_peer (&p1, "test_fs_unindex_data.conf"); - fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dstXXXXXX"); + fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst"); fs = GNUNET_FS_start (sched, cfg, "test-fs-unindex",