2 This file is part of GNUnet.
3 (C) 2001-2011 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - different priority for scheduling probe downloads?
29 #include "gnunet_constants.h"
30 #include "gnunet_fs_service.h"
34 #define DEBUG_DOWNLOAD GNUNET_NO
37 * Determine if the given download (options and meta data) should cause
38 * use to try to do a recursive download.
41 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
43 return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
44 ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
45 ( (dc->meta == NULL) &&
46 ( (NULL == dc->filename) ||
47 ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
49 strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
50 GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
55 * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
56 * only have to truncate the file once we're done).
58 * Given the offset of a block (with respect to the DBLOCKS) and its
59 * depth, return the offset where we would store this block in the
62 * @param fsize overall file size
63 * @param off offset of the block in the file
64 * @param depth depth of the block in the tree, 0 for DBLOCK
65 * @return off for DBLOCKS (depth == treedepth),
66 * otherwise an offset past the end
67 * of the file that does not overlap
68 * with the range for any other block
71 compute_disk_offset (uint64_t fsize,
76 uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
77 uint64_t loff; /* where do IBlocks for depth "i" start? */
78 unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
82 /* first IBlocks start at the end of file, rounded up
83 to full DBLOCK_SIZE */
84 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
85 lsize = ( (fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
86 GNUNET_assert (0 == (off % DBLOCK_SIZE));
87 ioff = (off / DBLOCK_SIZE);
91 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
92 GNUNET_assert (lsize > 0);
93 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
94 ioff /= CHK_PER_INODE;
96 return loff + ioff * sizeof (struct ContentHashKey);
101 * Given a block at the given offset and depth, calculate the offset
102 * for the CHK at the given index.
104 * @param offset the offset of the first
105 * DBLOCK in the subtree of the
107 * @param depth the depth of the IBLOCK in the tree, 0 for DBLOCK
108 * @param k which CHK in the IBLOCK are we
110 * @return offset if k=0, otherwise an appropriately
111 * larger value (i.e., if depth = 1,
112 * the returned value should be offset+k*DBLOCK_SIZE)
115 compute_dblock_offset (uint64_t offset,
120 uint64_t lsize; /* what is the size of the sum of all DBlocks
121 that a CHK at depth i corresponds to? */
126 for (i=1;i<depth;i++)
127 lsize *= CHK_PER_INODE;
128 return offset + k * lsize;
133 * Fill in all of the generic fields for a download event and call the
136 * @param pi structure to fill in
137 * @param dc overall download context
140 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
141 struct GNUNET_FS_DownloadContext *dc)
143 pi->value.download.dc = dc;
144 pi->value.download.cctx
146 pi->value.download.pctx
147 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
148 pi->value.download.sctx
149 = (dc->search == NULL) ? NULL : dc->search->client_info;
150 pi->value.download.uri
152 pi->value.download.filename
154 pi->value.download.size
156 pi->value.download.duration
157 = GNUNET_TIME_absolute_get_duration (dc->start_time);
158 pi->value.download.completed
160 pi->value.download.anonymity
162 pi->value.download.eta
163 = GNUNET_TIME_calculate_eta (dc->start_time,
166 pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
167 if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
168 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
171 dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
177 * We're ready to transmit a search request to the
178 * file-sharing service. Do it. If there is
179 * more than one request pending, try to send
180 * multiple or request another transmission.
183 * @param size number of bytes available in buf
184 * @param buf where the callee should write the message
185 * @return number of bytes written to buf
188 transmit_download_request (void *cls,
194 * Closure for iterator processing results.
196 struct ProcessResultClosure
202 GNUNET_HashCode query;
205 * Data found in P2P network.
210 * Our download context.
212 struct GNUNET_FS_DownloadContext *dc;
215 * Number of bytes in data.
222 enum GNUNET_BLOCK_Type type;
225 * Flag to indicate if this block should be stored on disk.
233 * Iterator over entries in the pending requests in the 'active' map for the
234 * reply that we just got.
236 * @param cls closure (our 'struct ProcessResultClosure')
237 * @param key query for the given value / request
238 * @param value value in the hash map (a 'struct DownloadRequest')
239 * @return GNUNET_YES (we should continue to iterate); unless serious error
242 process_result_with_request (void *cls,
243 const GNUNET_HashCode * key,
248 * We've found a matching block without downloading it.
249 * Encrypt it and pass it to our "receive" function as
250 * if we had received it from the network.
252 * @param dc download in question
253 * @param chk request this relates to
254 * @param dr request details
255 * @param block plaintext data matching request
256 * @param len number of bytes in block
257 * @param do_store should we still store the block on disk?
258 * @return GNUNET_OK on success
261 encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc,
262 const struct ContentHashKey *chk,
263 struct DownloadRequest *dr,
268 struct ProcessResultClosure prc;
270 struct GNUNET_CRYPTO_AesSessionKey sk;
271 struct GNUNET_CRYPTO_AesInitializationVector iv;
272 GNUNET_HashCode query;
274 GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
275 if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
281 return GNUNET_SYSERR;
283 GNUNET_CRYPTO_hash (enc, len, &query);
284 if (0 != memcmp (&query,
286 sizeof (GNUNET_HashCode)))
289 return GNUNET_SYSERR;
292 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
293 "Matching block for `%s' at offset %llu already present, no need for download!\n",
295 (unsigned long long) dr->offset);
297 /* already got it! */
301 prc.type = (0 == dr->depth)
302 ? GNUNET_BLOCK_TYPE_FS_DBLOCK
303 : GNUNET_BLOCK_TYPE_FS_IBLOCK;
304 prc.query = chk->query;
305 prc.do_store = do_store;
306 process_result_with_request (&prc,
314 * We've lost our connection with the FS service.
315 * Re-establish it and re-transmit all of our
318 * @param dc download context that is having trouble
321 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
325 * We found an entry in a directory. Check if the respective child
326 * already exists and if not create the respective child download.
328 * @param cls the parent download
329 * @param filename name of the file in the directory
330 * @param uri URI of the file (CHK or LOC)
331 * @param meta meta data of the file
332 * @param length number of bytes in data
333 * @param data contents of the file (or NULL if they were not inlined)
336 trigger_recursive_download (void *cls,
337 const char *filename,
338 const struct GNUNET_FS_Uri *uri,
339 const struct GNUNET_CONTAINER_MetaData *meta,
345 * We're done downloading a directory. Open the file and
346 * trigger all of the (remaining) child downloads.
348 * @param dc context of download that just completed
351 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
356 struct GNUNET_DISK_FileHandle *h;
357 struct GNUNET_DISK_MapHandle *m;
359 size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
360 size = (size_t) size64;
361 if (size64 != (uint64_t) size)
363 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
364 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
367 if (dc->filename != NULL)
369 h = GNUNET_DISK_file_open (dc->filename,
370 GNUNET_DISK_OPEN_READ,
371 GNUNET_DISK_PERM_NONE);
375 GNUNET_assert (dc->temp_filename != NULL);
376 h = GNUNET_DISK_file_open (dc->temp_filename,
377 GNUNET_DISK_OPEN_READ,
378 GNUNET_DISK_PERM_NONE);
382 data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
385 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
386 _("Directory too large for system address space\n"));
390 GNUNET_FS_directory_list_contents (size,
393 &trigger_recursive_download,
395 GNUNET_DISK_file_unmap (m);
397 GNUNET_DISK_file_close (h);
398 if (dc->filename == NULL)
400 if (0 != UNLINK (dc->temp_filename))
401 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
404 GNUNET_free (dc->temp_filename);
405 dc->temp_filename = NULL;
411 * Check if all child-downloads have completed (or trigger them if
412 * necessary) and once we're completely done, signal completion (and
413 * possibly recurse to parent). This function MUST be called when the
414 * download of a file itself is done or when the download of a file is
415 * done and then later a direct child download has completed (and
416 * hence this download may complete itself).
418 * @param dc download to check for completion of children
421 check_completed (struct GNUNET_FS_DownloadContext *dc)
423 struct GNUNET_FS_ProgressInfo pi;
424 struct GNUNET_FS_DownloadContext *pos;
426 /* first, check if we need to download children */
427 if ( (dc->child_head == NULL) &&
428 (is_recursive_download (dc)) )
429 full_recursive_download (dc);
430 /* then, check if children are done already */
431 pos = dc->child_head;
434 if ( (pos->emsg == NULL) &&
435 (pos->completed < pos->length) )
436 return; /* not done yet */
437 if ( (pos->child_head != NULL) &&
438 (pos->has_finished != GNUNET_YES) )
439 return; /* not transitively done yet */
442 /* All of our children are done, so mark this download done */
443 dc->has_finished = GNUNET_YES;
444 if (dc->job_queue != NULL)
446 GNUNET_FS_dequeue_ (dc->job_queue);
447 dc->job_queue = NULL;
449 GNUNET_FS_download_sync_ (dc);
451 /* signal completion */
452 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
453 GNUNET_FS_download_make_status_ (&pi, dc);
455 /* let parent know */
456 if (dc->parent != NULL)
457 check_completed (dc->parent);
462 * We got a block of plaintext data (from the meta data).
463 * Try it for upward reconstruction of the data. On success,
464 * the top-level block will move to state BRS_DOWNLOAD_UP.
466 * @param dc context for the download
467 * @param dr download request to match against
468 * @param data plaintext data, starting from the beginning of the file
469 * @param data_len number of bytes in data
472 try_match_block (struct GNUNET_FS_DownloadContext *dc,
473 struct DownloadRequest *dr,
477 struct GNUNET_FS_ProgressInfo pi;
479 char enc[DBLOCK_SIZE];
480 struct ContentHashKey chks[CHK_PER_INODE];
481 struct ContentHashKey in_chk;
482 struct GNUNET_CRYPTO_AesSessionKey sk;
483 struct GNUNET_CRYPTO_AesInitializationVector iv;
485 struct DownloadRequest *drc;
486 struct GNUNET_DISK_FileHandle *fh;
493 odata_len = data_len;
494 if (BRS_DOWNLOAD_UP == dr->state)
498 complete = GNUNET_YES;
499 for (i=0;i<dr->num_children;i++)
501 drc = dr->children[i];
505 if (drc->state != BRS_RECONSTRUCT_META_UP)
506 complete = GNUNET_NO;
510 if (GNUNET_YES != complete)
512 data = (const char*) chks;
513 dlen = dr->num_children * sizeof (struct ContentHashKey);
517 if (dr->offset > data_len)
519 dlen = GNUNET_MIN (data_len - dr->offset,
522 GNUNET_CRYPTO_hash (&data[dr->offset],
525 GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
526 if (-1 == GNUNET_CRYPTO_aes_encrypt (&data[dr->offset], dlen,
534 GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
539 dr->state = BRS_RECONSTRUCT_META_UP;
542 if (0 != memcmp (&in_chk,
544 sizeof (struct ContentHashKey)))
546 /* other peer provided bogus meta data */
550 /* write block to disk */
551 fn = dc->filename != NULL
554 fh = GNUNET_DISK_file_open (fn,
555 GNUNET_DISK_OPEN_READWRITE |
556 GNUNET_DISK_OPEN_CREATE |
557 GNUNET_DISK_OPEN_TRUNCATE,
558 GNUNET_DISK_PERM_USER_READ |
559 GNUNET_DISK_PERM_USER_WRITE |
560 GNUNET_DISK_PERM_GROUP_READ |
561 GNUNET_DISK_PERM_OTHER_READ);
564 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
567 GNUNET_asprintf (&dc->emsg,
568 _("Failed to open file `%s' for writing"),
570 GNUNET_DISK_file_close (fh);
571 dr->state = BRS_ERROR;
572 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
573 pi.value.download.specifics.error.message = dc->emsg;
574 GNUNET_FS_download_make_status_ (&pi, dc);
578 GNUNET_DISK_file_write (fh,
582 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
585 GNUNET_asprintf (&dc->emsg,
586 _("Failed to open file `%s' for writing"),
588 GNUNET_DISK_file_close (fh);
589 dr->state = BRS_ERROR;
590 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
591 pi.value.download.specifics.error.message = dc->emsg;
592 GNUNET_FS_download_make_status_ (&pi, dc);
595 GNUNET_DISK_file_close (fh);
597 dr->state = BRS_DOWNLOAD_UP;
598 dc->completed = dc->length;
599 GNUNET_FS_download_sync_ (dc);
600 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
601 pi.value.download.specifics.progress.data = data;
602 pi.value.download.specifics.progress.offset = 0;
603 pi.value.download.specifics.progress.data_len = dlen;
604 pi.value.download.specifics.progress.depth = 0;
605 GNUNET_FS_download_make_status_ (&pi, dc);
606 if ( (NULL != dc->filename) &&
607 (0 != truncate (dc->filename,
608 GNUNET_ntohll (dc->uri->data.chk.file_length))) )
609 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
612 check_completed (dc);
615 /* how did we get here? */
623 * Type of a function that libextractor calls for each
624 * meta data item found. If we find full data meta data,
625 * call 'try_match_block' on it.
627 * @param cls our 'struct GNUNET_FS_DownloadContext*'
628 * @param plugin_name name of the plugin that produced this value;
629 * special values can be used (i.e. '<zlib>' for zlib being
630 * used in the main libextractor library and yielding
632 * @param type libextractor-type describing the meta data
633 * @param format basic format information about data
634 * @param data_mime_type mime-type of data (not of the original file);
635 * can be NULL (if mime-type is not known)
636 * @param data actual meta-data found
637 * @param data_len number of bytes in data
638 * @return 0 to continue extracting, 1 to abort
641 match_full_data (void *cls,
642 const char *plugin_name,
643 enum EXTRACTOR_MetaType type,
644 enum EXTRACTOR_MetaFormat format,
645 const char *data_mime_type,
649 struct GNUNET_FS_DownloadContext *dc = cls;
651 if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
654 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
655 "Found %u bytes of FD!\n",
656 (unsigned int) data_len);
658 if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
661 return 1; /* bogus meta data */
672 * Set the state of the given download request to
673 * BRS_DOWNLOAD_UP and propagate it up the tree.
675 * @param dr download request that is done
678 propagate_up (struct DownloadRequest *dr)
684 dr->state = BRS_DOWNLOAD_UP;
688 for (i=0;i<dr->num_children;i++)
689 if (dr->children[i]->state != BRS_DOWNLOAD_UP)
692 while (i == dr->num_children);
697 * Try top-down reconstruction. Before, the given request node
698 * must have the state BRS_CHK_SET. Afterwards, more nodes may
699 * have that state or advanced to BRS_DOWNLOAD_DOWN or even
700 * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the
703 * @param dc overall download this block belongs to
704 * @param dr block to reconstruct
707 try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
708 struct DownloadRequest *dr)
711 char block[DBLOCK_SIZE];
716 unsigned int chk_off;
717 struct DownloadRequest *drc;
718 uint64_t child_block_size;
719 const struct ContentHashKey *chks;
722 GNUNET_assert (dc->rfh != NULL);
723 GNUNET_assert (dr->state == BRS_CHK_SET);
724 total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
725 GNUNET_assert (dr->depth < dc->treedepth);
726 len = GNUNET_FS_tree_calculate_block_size (total,
729 GNUNET_assert (len <= DBLOCK_SIZE);
730 off = compute_disk_offset (total,
733 if (dc->old_file_size < off + len)
734 return; /* failure */
736 GNUNET_DISK_file_seek (dc->rfh,
738 GNUNET_DISK_SEEK_SET) )
740 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
743 return; /* failure */
746 GNUNET_DISK_file_read (dc->rfh,
750 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
753 return; /* failure */
755 GNUNET_CRYPTO_hash (block, len, &key);
756 if (0 != memcmp (&key,
758 sizeof (GNUNET_HashCode)))
759 return; /* mismatch */
761 encrypt_existing_match (dc,
768 /* hash matches but encrypted block does not, really bad */
769 dr->state = BRS_ERROR;
771 while (dr->parent != NULL)
774 dr->state = BRS_ERROR;
779 dr->state = BRS_DOWNLOAD_DOWN;
781 /* set CHKs for children */
782 up_done = GNUNET_YES;
783 chks = (const struct ContentHashKey*) block;
784 for (i=0;i<dr->num_children;i++)
786 drc = dr->children[i];
787 GNUNET_assert (drc->offset >= dr->offset);
788 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
789 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
790 chk_off = (drc->offset - dr->offset) / child_block_size;
791 GNUNET_assert (drc->state == BRS_INIT);
792 drc->state = BRS_CHK_SET;
793 drc->chk = chks[chk_off];
794 try_top_down_reconstruction (dc, drc);
795 if (drc->state != BRS_DOWNLOAD_UP)
796 up_done = GNUNET_NO; /* children not all done */
798 if (up_done == GNUNET_YES)
799 propagate_up (dr); /* children all done (or no children...) */
804 * Schedule the download of the specified block in the tree.
806 * @param dc overall download this block belongs to
807 * @param dr request to schedule
810 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
811 struct DownloadRequest *dr)
820 case BRS_RECONSTRUCT_DOWN:
823 case BRS_RECONSTRUCT_META_UP:
826 case BRS_RECONSTRUCT_UP:
830 /* normal case, start download */
832 case BRS_DOWNLOAD_DOWN:
833 for (i=0;i<dr->num_children;i++)
834 schedule_block_download (dc, dr->children[i]);
836 case BRS_DOWNLOAD_UP:
844 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
845 "Scheduling download at offset %llu and depth %u for `%s'\n",
846 (unsigned long long) dr->offset,
848 GNUNET_h2s (&dr->chk.query));
850 GNUNET_assert (GNUNET_NO ==
851 GNUNET_CONTAINER_multihashmap_contains_value (dc->active,
854 GNUNET_CONTAINER_multihashmap_put (dc->active,
857 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
858 if (dc->client == NULL)
859 return; /* download not active */
860 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
863 dr->is_pending = GNUNET_YES;
865 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
866 sizeof (struct SearchMessage),
867 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
869 &transmit_download_request,
874 #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
877 * We found an entry in a directory. Check if the respective child
878 * already exists and if not create the respective child download.
880 * @param cls the parent download
881 * @param filename name of the file in the directory
882 * @param uri URI of the file (CHK or LOC)
883 * @param meta meta data of the file
884 * @param length number of bytes in data
885 * @param data contents of the file (or NULL if they were not inlined)
888 trigger_recursive_download (void *cls,
889 const char *filename,
890 const struct GNUNET_FS_Uri *uri,
891 const struct GNUNET_CONTAINER_MetaData *meta,
895 struct GNUNET_FS_DownloadContext *dc = cls;
896 struct GNUNET_FS_DownloadContext *cpos;
907 return; /* entry for the directory itself */
908 cpos = dc->child_head;
911 if ( (GNUNET_FS_uri_test_equal (uri,
913 ( (filename != NULL) &&
914 (0 == strcmp (cpos->filename,
920 return; /* already exists */
922 if (NULL == filename)
924 fn = GNUNET_FS_meta_data_suggest_filename (meta);
927 us = GNUNET_FS_uri_to_string (uri);
928 fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_CHK_PREFIX)]);
931 else if (fn[0] == '.')
934 us = GNUNET_FS_uri_to_string (uri);
935 GNUNET_asprintf (&fn,
937 &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], ext);
941 /* change '\' to '/' (this should have happened
942 during insertion, but malicious peers may
943 not have done this) */
944 while (NULL != (pos = strstr (fn, "\\")))
946 /* remove '../' everywhere (again, well-behaved
947 peers don't do this, but don't trust that
948 we did not get something nasty) */
949 while (NULL != (pos = strstr (fn, "../")))
957 if (dc->filename == NULL)
963 dn = GNUNET_strdup (dc->filename);
964 GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
966 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
967 GNUNET_FS_DIRECTORY_EXT)) );
968 sfn = GNUNET_strdup (filename);
969 while ( (strlen (sfn) > 0) &&
970 (filename[strlen(sfn)-1] == '/') )
971 sfn[strlen(sfn)-1] = '\0';
972 if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
974 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
975 GNUNET_FS_DIRECTORY_EXT)) )
976 dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
977 if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
978 ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
980 strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
981 GNUNET_FS_DIRECTORY_EXT)) ) )
983 GNUNET_asprintf (&full_name,
988 GNUNET_FS_DIRECTORY_EXT);
992 GNUNET_asprintf (&full_name,
1000 if ( (full_name != NULL) &&
1002 GNUNET_DISK_directory_create_for_file (full_name)) )
1004 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1005 _("Failed to create directory for recursive download of `%s'\n"),
1007 GNUNET_free (full_name);
1008 GNUNET_free_non_null (fn);
1014 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1015 "Triggering recursive download of size %llu with %u bytes MD\n",
1016 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
1017 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (meta));
1019 GNUNET_FS_download_start (dc->h,
1022 full_name, temp_name,
1024 GNUNET_FS_uri_chk_get_file_size (uri),
1029 GNUNET_free_non_null (full_name);
1030 GNUNET_free_non_null (temp_name);
1031 GNUNET_free_non_null (fn);
1036 * (recursively) free download request structure
1038 * @param dr request to free
1041 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
1047 for (i=0;i<dr->num_children;i++)
1048 GNUNET_FS_free_download_request_ (dr->children[i]);
1049 GNUNET_free_non_null (dr->children);
1055 * Iterator over entries in the pending requests in the 'active' map for the
1056 * reply that we just got.
1058 * @param cls closure (our 'struct ProcessResultClosure')
1059 * @param key query for the given value / request
1060 * @param value value in the hash map (a 'struct DownloadRequest')
1061 * @return GNUNET_YES (we should continue to iterate); unless serious error
1064 process_result_with_request (void *cls,
1065 const GNUNET_HashCode *key,
1068 struct ProcessResultClosure *prc = cls;
1069 struct DownloadRequest *dr = value;
1070 struct GNUNET_FS_DownloadContext *dc = prc->dc;
1071 struct DownloadRequest *drc;
1072 struct GNUNET_DISK_FileHandle *fh = NULL;
1073 struct GNUNET_CRYPTO_AesSessionKey skey;
1074 struct GNUNET_CRYPTO_AesInitializationVector iv;
1076 struct GNUNET_FS_ProgressInfo pi;
1081 struct ContentHashKey *chk;
1084 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1085 "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n",
1088 (unsigned long long) dr->offset,
1089 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1092 bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
1095 if (prc->size != bs)
1097 GNUNET_asprintf (&dc->emsg,
1098 _("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"),
1101 (unsigned long long) dr->offset,
1102 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
1104 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1107 while (dr->parent != NULL)
1109 dr->state = BRS_ERROR;
1112 dr->state = BRS_ERROR;
1116 (void) GNUNET_CONTAINER_multihashmap_remove (dc->active,
1119 if (GNUNET_YES == dr->is_pending)
1121 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1124 dr->is_pending = GNUNET_NO;
1128 GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
1129 if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
1136 dc->emsg = GNUNET_strdup (_("internal error decrypting content"));
1139 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
1143 if ( ( GNUNET_YES == prc->do_store) &&
1144 ( (dc->filename != NULL) ||
1145 (is_recursive_download (dc)) ) &&
1146 ( (dr->depth == dc->treedepth) ||
1147 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
1149 fh = GNUNET_DISK_file_open (dc->filename != NULL
1151 : dc->temp_filename,
1152 GNUNET_DISK_OPEN_READWRITE |
1153 GNUNET_DISK_OPEN_CREATE,
1154 GNUNET_DISK_PERM_USER_READ |
1155 GNUNET_DISK_PERM_USER_WRITE |
1156 GNUNET_DISK_PERM_GROUP_READ |
1157 GNUNET_DISK_PERM_OTHER_READ);
1160 GNUNET_asprintf (&dc->emsg,
1161 _("Download failed: could not open file `%s': %s\n"),
1167 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1168 "Saving decrypted block to disk at offset %llu\n",
1169 (unsigned long long) off);
1172 GNUNET_DISK_file_seek (fh,
1174 GNUNET_DISK_SEEK_SET) ) )
1176 GNUNET_asprintf (&dc->emsg,
1177 _("Failed to seek to offset %llu in file `%s': %s\n"),
1178 (unsigned long long) off,
1184 GNUNET_DISK_file_write (fh,
1188 GNUNET_asprintf (&dc->emsg,
1189 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1190 (unsigned int) prc->size,
1191 (unsigned long long) off,
1196 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1202 /* DBLOCK, update progress and try recursion if applicable */
1204 if (dr->offset < dc->offset)
1206 /* starting offset begins in the middle of pt,
1207 do not count first bytes as progress */
1208 GNUNET_assert (app > (dc->offset - dr->offset));
1209 app -= (dc->offset - dr->offset);
1211 if (dr->offset + prc->size > dc->offset + dc->length)
1213 /* end of block is after relevant range,
1214 do not count last bytes as progress */
1215 GNUNET_assert (app > (dr->offset + prc->size) - (dc->offset + dc->length));
1216 app -= (dr->offset + prc->size) - (dc->offset + dc->length);
1218 dc->completed += app;
1220 /* do recursive download if option is set and either meta data
1221 says it is a directory or if no meta data is given AND filename
1222 ends in '.gnd' (top-level case) */
1223 if (is_recursive_download (dc))
1224 GNUNET_FS_directory_list_contents (prc->size,
1227 &trigger_recursive_download,
1231 dr->state = BRS_DOWNLOAD_DOWN;
1232 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1233 pi.value.download.specifics.progress.data = pt;
1234 pi.value.download.specifics.progress.offset = dr->offset;
1235 pi.value.download.specifics.progress.data_len = prc->size;
1236 pi.value.download.specifics.progress.depth = dr->depth;
1237 GNUNET_FS_download_make_status_ (&pi, dc);
1238 GNUNET_assert (dc->completed <= dc->length);
1242 if (dc->completed == dc->length)
1244 /* download completed, signal */
1246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1247 "Download completed, truncating file to desired length %llu\n",
1248 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1250 /* truncate file to size (since we store IBlocks at the end) */
1251 if (dc->filename != NULL)
1253 if (0 != truncate (dc->filename,
1254 GNUNET_ntohll (dc->uri->data.chk.file_length)))
1255 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1259 GNUNET_assert (dr->depth == 0);
1260 check_completed (dc);
1264 /* bottom of the tree, no child downloads possible, just sync */
1265 GNUNET_FS_download_sync_ (dc);
1270 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1271 "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1273 (unsigned long long) dr->offset);
1275 GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1276 chk = (struct ContentHashKey*) pt;
1277 for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1279 off = compute_dblock_offset (dr->offset,
1282 drc = dr->children[i];
1287 drc->state = BRS_CHK_SET;
1288 schedule_block_download (dc, drc);
1290 case BRS_RECONSTRUCT_DOWN:
1293 case BRS_RECONSTRUCT_META_UP:
1296 case BRS_RECONSTRUCT_UP:
1302 case BRS_DOWNLOAD_DOWN:
1305 case BRS_DOWNLOAD_UP:
1316 GNUNET_FS_download_sync_ (dc);
1321 GNUNET_DISK_file_close (fh);
1322 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1323 pi.value.download.specifics.error.message = dc->emsg;
1324 GNUNET_FS_download_make_status_ (&pi, dc);
1325 /* abort all pending requests */
1328 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1331 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1333 GNUNET_FS_free_download_request_ (dc->top_request);
1334 dc->top_request = NULL;
1335 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1337 dc->pending_head = NULL;
1338 dc->pending_tail = NULL;
1339 GNUNET_FS_download_sync_ (dc);
1345 * Process a download result.
1347 * @param dc our download context
1348 * @param type type of the result
1349 * @param data the (encrypted) response
1350 * @param size size of data
1353 process_result (struct GNUNET_FS_DownloadContext *dc,
1354 enum GNUNET_BLOCK_Type type,
1358 struct ProcessResultClosure prc;
1364 prc.do_store = GNUNET_YES;
1365 GNUNET_CRYPTO_hash (data, size, &prc.query);
1367 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1368 "Received result for query `%s' from `%s'-service\n",
1369 GNUNET_h2s (&prc.query),
1372 GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1374 &process_result_with_request,
1380 * Type of a function to call when we receive a message
1383 * @param cls closure
1384 * @param msg message received, NULL on timeout or fatal error
1387 receive_results (void *cls,
1388 const struct GNUNET_MessageHeader * msg)
1390 struct GNUNET_FS_DownloadContext *dc = cls;
1391 const struct PutMessage *cm;
1394 if ( (NULL == msg) ||
1395 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1396 (sizeof (struct PutMessage) > ntohs(msg->size)) )
1398 GNUNET_break (msg == NULL);
1402 msize = ntohs(msg->size);
1403 cm = (const struct PutMessage*) msg;
1407 msize - sizeof (struct PutMessage));
1408 if (dc->client == NULL)
1409 return; /* fatal error */
1410 /* continue receiving */
1411 GNUNET_CLIENT_receive (dc->client,
1414 GNUNET_TIME_UNIT_FOREVER_REL);
1420 * We're ready to transmit a search request to the
1421 * file-sharing service. Do it. If there is
1422 * more than one request pending, try to send
1423 * multiple or request another transmission.
1425 * @param cls closure
1426 * @param size number of bytes available in buf
1427 * @param buf where the callee should write the message
1428 * @return number of bytes written to buf
1431 transmit_download_request (void *cls,
1435 struct GNUNET_FS_DownloadContext *dc = cls;
1437 struct SearchMessage *sm;
1438 struct DownloadRequest *dr;
1444 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1445 "Transmitting download request failed, trying to reconnect\n");
1450 GNUNET_assert (size >= sizeof (struct SearchMessage));
1453 while ( (NULL != (dr = dc->pending_head)) &&
1454 (size >= msize + sizeof (struct SearchMessage)) )
1457 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1458 "Transmitting download request for `%s' to `%s'-service\n",
1459 GNUNET_h2s (&dr->chk.query),
1462 memset (sm, 0, sizeof (struct SearchMessage));
1463 sm->header.size = htons (sizeof (struct SearchMessage));
1464 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1465 if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1466 sm->options = htonl (1);
1468 sm->options = htonl (0);
1470 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
1472 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
1473 sm->anonymity_level = htonl (dc->anonymity);
1474 sm->target = dc->target.hashPubKey;
1475 sm->query = dr->chk.query;
1476 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1479 dr->is_pending = GNUNET_NO;
1480 msize += sizeof (struct SearchMessage);
1483 if (dc->pending_head != NULL)
1485 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1486 sizeof (struct SearchMessage),
1487 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1489 &transmit_download_request,
1491 GNUNET_assert (dc->th != NULL);
1498 * Reconnect to the FS service and transmit our queries NOW.
1500 * @param cls our download context
1504 do_reconnect (void *cls,
1505 const struct GNUNET_SCHEDULER_TaskContext *tc)
1507 struct GNUNET_FS_DownloadContext *dc = cls;
1508 struct GNUNET_CLIENT_Connection *client;
1510 dc->task = GNUNET_SCHEDULER_NO_TASK;
1511 client = GNUNET_CLIENT_connect ("fs",
1515 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1516 "Connecting to `%s'-service failed, will try again.\n",
1521 dc->client = client;
1522 if (dc->pending_head != NULL)
1524 dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1525 sizeof (struct SearchMessage),
1526 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1528 &transmit_download_request,
1530 GNUNET_assert (dc->th != NULL);
1532 GNUNET_CLIENT_receive (client,
1535 GNUNET_TIME_UNIT_FOREVER_REL);
1540 * Add entries to the pending list.
1542 * @param cls our download context
1544 * @param entry entry of type "struct DownloadRequest"
1548 retry_entry (void *cls,
1549 const GNUNET_HashCode *key,
1552 struct GNUNET_FS_DownloadContext *dc = cls;
1553 struct DownloadRequest *dr = entry;
1557 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
1560 dr->is_pending = GNUNET_YES;
1566 * We've lost our connection with the FS service.
1567 * Re-establish it and re-transmit all of our
1570 * @param dc download context that is having trouble
1573 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1576 if (NULL != dc->client)
1579 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1580 "Moving all requests back to pending list\n");
1584 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1587 /* full reset of the pending list */
1588 dc->pending_head = NULL;
1589 dc->pending_tail = NULL;
1590 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1593 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1597 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1598 "Will try to reconnect in 1s\n");
1601 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1608 * We're allowed to ask the FS service for our blocks. Start the download.
1610 * @param cls the 'struct GNUNET_FS_DownloadContext'
1611 * @param client handle to use for communcation with FS (we must destroy it!)
1614 activate_fs_download (void *cls,
1615 struct GNUNET_CLIENT_Connection *client)
1617 struct GNUNET_FS_DownloadContext *dc = cls;
1618 struct GNUNET_FS_ProgressInfo pi;
1621 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1622 "Download activated\n");
1624 GNUNET_assert (NULL != client);
1625 GNUNET_assert (dc->client == NULL);
1626 GNUNET_assert (dc->th == NULL);
1627 dc->client = client;
1628 GNUNET_CLIENT_receive (client,
1631 GNUNET_TIME_UNIT_FOREVER_REL);
1632 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1633 GNUNET_FS_download_make_status_ (&pi, dc);
1634 GNUNET_assert (dc->pending_head == NULL);
1635 GNUNET_assert (dc->pending_tail == NULL);
1636 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1640 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1641 "Asking for transmission to FS service\n");
1643 if (dc->pending_head != NULL)
1645 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1646 sizeof (struct SearchMessage),
1647 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1649 &transmit_download_request,
1651 GNUNET_assert (dc->th != NULL);
1657 * We must stop to ask the FS service for our blocks. Pause the download.
1659 * @param cls the 'struct GNUNET_FS_DownloadContext'
1662 deactivate_fs_download (void *cls)
1664 struct GNUNET_FS_DownloadContext *dc = cls;
1665 struct GNUNET_FS_ProgressInfo pi;
1668 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1669 "Download deactivated\n");
1673 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1676 if (NULL != dc->client)
1678 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1681 dc->pending_head = NULL;
1682 dc->pending_tail = NULL;
1683 pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1684 GNUNET_FS_download_make_status_ (&pi, dc);
1689 * (recursively) Create a download request structure.
1691 * @param parent parent of the current entry
1692 * @param depth depth of the current entry, 0 are the DBLOCKs,
1693 * top level block is 'dc->treedepth - 1'
1694 * @param dr_offset offset in the original file this block maps to
1695 * (as in, offset of the first byte of the first DBLOCK
1696 * in the subtree rooted in the returned download request tree)
1697 * @param file_start_offset desired starting offset for the download
1698 * in the original file; requesting tree should not contain
1699 * DBLOCKs prior to the file_start_offset
1700 * @param desired_length desired number of bytes the user wanted to access
1701 * (from file_start_offset). Resulting tree should not contain
1702 * DBLOCKs after file_start_offset + file_length.
1703 * @return download request tree for the given range of DBLOCKs at
1704 * the specified depth
1706 static struct DownloadRequest *
1707 create_download_request (struct DownloadRequest *parent,
1710 uint64_t file_start_offset,
1711 uint64_t desired_length)
1713 struct DownloadRequest *dr;
1715 unsigned int head_skip;
1716 uint64_t child_block_size;
1718 dr = GNUNET_malloc (sizeof (struct DownloadRequest));
1719 dr->parent = parent;
1721 dr->offset = dr_offset;
1724 child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
1726 /* calculate how many blocks at this level are not interesting
1727 from the start (rounded down), either because of the requested
1728 file offset or because this IBlock is further along */
1729 if (dr_offset < file_start_offset)
1730 head_skip = file_start_offset / child_block_size;
1732 head_skip = dr_offset / child_block_size;
1734 /* calculate index of last block at this level that is interesting (rounded up) */
1735 dr->num_children = file_start_offset + desired_length / child_block_size;
1736 if (dr->num_children * child_block_size < file_start_offset + desired_length)
1737 dr->num_children++; /* round up */
1739 /* now we can get the total number of children for this block */
1740 dr->num_children -= head_skip;
1741 if (dr->num_children > CHK_PER_INODE)
1742 dr->num_children = CHK_PER_INODE; /* cap at max */
1744 /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
1745 GNUNET_assert (dr->num_children > 0);
1747 dr->children = GNUNET_malloc (dr->num_children *
1748 sizeof (struct DownloadRequest *));
1749 for (i=0;i<dr->num_children;i++)
1750 dr->children[i] = create_download_request (dr,
1752 dr_offset + i * child_block_size,
1761 * Continuation after a possible attempt to reconstruct
1762 * the current IBlock from the existing file.
1764 * @param cls the 'struct ReconstructContext'
1765 * @param tc scheduler context
1768 reconstruct_cont (void *cls,
1769 const struct GNUNET_SCHEDULER_TaskContext *tc)
1771 struct GNUNET_FS_DownloadContext *dc = cls;
1773 /* clean up state from tree encoder */
1776 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
1779 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
1781 GNUNET_SCHEDULER_cancel (dc->task);
1782 dc->task = GNUNET_SCHEDULER_NO_TASK;
1784 if (dc->rfh != NULL)
1786 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
1789 /* start "normal" download */
1790 schedule_block_download (dc,
1796 * Task requesting the next block from the tree encoder.
1798 * @param cls the 'struct GNUJNET_FS_DownloadContext' we're processing
1799 * @param tc task context
1802 get_next_block (void *cls,
1803 const struct GNUNET_SCHEDULER_TaskContext *tc)
1805 struct GNUNET_FS_DownloadContext *dc = cls;
1807 dc->task = GNUNET_SCHEDULER_NO_TASK;
1808 GNUNET_FS_tree_encoder_next (dc->te);
1814 * Function called asking for the current (encoded)
1815 * block to be processed. After processing the
1816 * client should either call "GNUNET_FS_tree_encode_next"
1817 * or (on error) "GNUNET_FS_tree_encode_finish".
1819 * This function checks if the content on disk matches
1820 * the expected content based on the URI.
1822 * @param cls closure
1823 * @param chk content hash key for the block
1824 * @param offset offset of the block
1825 * @param depth depth of the block, 0 for DBLOCK
1826 * @param type type of the block (IBLOCK or DBLOCK)
1827 * @param block the (encrypted) block
1828 * @param block_size size of block (in bytes)
1831 reconstruct_cb (void *cls,
1832 const struct ContentHashKey *chk,
1835 enum GNUNET_BLOCK_Type type,
1837 uint16_t block_size)
1839 struct GNUNET_FS_DownloadContext *dc = cls;
1840 struct GNUNET_FS_ProgressInfo pi;
1841 struct DownloadRequest *dr;
1845 /* find corresponding request entry */
1846 dr = dc->top_request;
1847 while (dr->depth > depth)
1849 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1850 chld = (offset - dr->offset) / blen;
1851 GNUNET_assert (chld < dr->num_children);
1852 dr = dr->children[chld];
1858 case BRS_RECONSTRUCT_DOWN:
1860 case BRS_RECONSTRUCT_META_UP:
1862 case BRS_RECONSTRUCT_UP:
1865 if (0 == memcmp (chk,
1867 sizeof (struct ContentHashKey)))
1869 /* block matches, hence tree below matches;
1870 this request is done! */
1871 dr->state = BRS_DOWNLOAD_UP;
1872 /* calculate how many bytes of payload this block
1874 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1875 /* how many of those bytes are in the requested range? */
1876 blen = GNUNET_MIN (blen,
1877 dc->length + dc->offset - dr->offset);
1878 /* signal progress */
1879 dc->completed += blen;
1880 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1881 pi.value.download.specifics.progress.data = NULL;
1882 pi.value.download.specifics.progress.offset = offset;
1883 pi.value.download.specifics.progress.data_len = 0;
1884 pi.value.download.specifics.progress.depth = 0;
1885 GNUNET_FS_download_make_status_ (&pi, dc);
1891 case BRS_DOWNLOAD_DOWN:
1893 case BRS_DOWNLOAD_UP:
1901 if ( (dr == dc->top_request) &&
1902 (dr->state == BRS_DOWNLOAD_UP) )
1904 check_completed (dc);
1907 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
1913 * Function called by the tree encoder to obtain a block of plaintext
1914 * data (for the lowest level of the tree).
1916 * @param cls our 'struct ReconstructContext'
1917 * @param offset identifies which block to get
1918 * @param max (maximum) number of bytes to get; returning
1919 * fewer will also cause errors
1920 * @param buf where to copy the plaintext buffer
1921 * @param emsg location to store an error message (on error)
1922 * @return number of bytes copied to buf, 0 on error
1925 fh_reader (void *cls,
1931 struct GNUNET_FS_DownloadContext *dc = cls;
1932 struct GNUNET_DISK_FileHandle *fh = dc->rfh;
1937 GNUNET_DISK_file_seek (fh,
1939 GNUNET_DISK_SEEK_SET))
1941 *emsg = GNUNET_strdup (strerror (errno));
1944 ret = GNUNET_DISK_file_read (fh, buf, max);
1947 *emsg = GNUNET_strdup (strerror (errno));
1955 * Task that creates the initial (top-level) download
1956 * request for the file.
1958 * @param cls the 'struct GNUNET_FS_DownloadContext'
1959 * @param tc scheduler context
1962 GNUNET_FS_download_start_task_ (void *cls,
1963 const struct GNUNET_SCHEDULER_TaskContext *tc)
1965 struct GNUNET_FS_DownloadContext *dc = cls;
1966 struct GNUNET_FS_ProgressInfo pi;
1967 struct GNUNET_DISK_FileHandle *fh;
1970 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1971 "Start task running...\n");
1973 dc->task = GNUNET_SCHEDULER_NO_TASK;
1974 if (dc->length == 0)
1976 /* no bytes required! */
1977 if (dc->filename != NULL)
1979 fh = GNUNET_DISK_file_open (dc->filename,
1980 GNUNET_DISK_OPEN_READWRITE |
1981 GNUNET_DISK_OPEN_CREATE |
1982 ( (0 == GNUNET_FS_uri_chk_get_file_size (dc->uri))
1983 ? GNUNET_DISK_OPEN_TRUNCATE : 0),
1984 GNUNET_DISK_PERM_USER_READ |
1985 GNUNET_DISK_PERM_USER_WRITE |
1986 GNUNET_DISK_PERM_GROUP_READ |
1987 GNUNET_DISK_PERM_OTHER_READ);
1988 GNUNET_DISK_file_close (fh);
1990 GNUNET_FS_download_sync_ (dc);
1991 check_completed (dc);
1994 if (dc->emsg != NULL)
1996 if (dc->top_request == NULL)
1998 dc->top_request = create_download_request (NULL, dc->treedepth - 1, 0,
1999 dc->offset, dc->length);
2000 dc->top_request->state = BRS_CHK_SET;
2001 dc->top_request->chk = (dc->uri->type == chk)
2002 ? dc->uri->data.chk.chk
2003 : dc->uri->data.loc.fi.chk;
2005 GNUNET_FS_download_sync_ (dc);
2006 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
2007 pi.value.download.specifics.start.meta = dc->meta;
2008 GNUNET_FS_download_make_status_ (&pi, dc);
2010 GNUNET_FS_download_start_downloading_ (dc);
2011 /* attempt reconstruction from disk */
2012 if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
2013 dc->rfh = GNUNET_DISK_file_open (dc->filename,
2014 GNUNET_DISK_OPEN_READ,
2015 GNUNET_DISK_PERM_NONE);
2016 if (dc->top_request->state == BRS_CHK_SET)
2018 if (dc->rfh != NULL)
2020 /* first, try top-down */
2022 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2023 "Trying top-down reconstruction for `%s'\n",
2026 try_top_down_reconstruction (dc, dc->top_request);
2027 switch (dc->top_request->state)
2031 case BRS_DOWNLOAD_DOWN:
2032 break; /* normal, some blocks already down */
2033 case BRS_DOWNLOAD_UP:
2034 /* already done entirely, party! */
2035 if (dc->rfh != NULL)
2037 /* avoid hanging on to file handle longer than
2039 GNUNET_DISK_file_close (dc->rfh);
2044 GNUNET_asprintf (&dc->emsg,
2046 GNUNET_FS_download_sync_ (dc);
2047 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
2048 pi.value.download.specifics.error.message = dc->emsg;
2049 GNUNET_FS_download_make_status_ (&pi, dc);
2057 /* attempt reconstruction from meta data */
2058 if ( (GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) &&
2059 (NULL != dc->meta) )
2062 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2063 "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
2064 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
2065 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (dc->meta));
2067 GNUNET_CONTAINER_meta_data_iterate (dc->meta,
2070 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2072 if (dc->rfh != NULL)
2074 /* avoid hanging on to file handle longer than
2076 GNUNET_DISK_file_close (dc->rfh);
2079 return; /* finished, status update was already done for us */
2082 if (dc->rfh != NULL)
2084 /* finally, try bottom-up */
2086 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2087 "Trying bottom-up reconstruction of file `%s'\n",
2090 dc->te = GNUNET_FS_tree_encoder_create (dc->h,
2097 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
2102 /* simple, top-level download */
2103 schedule_block_download (dc,
2106 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2107 check_completed (dc);
2112 * Create SUSPEND event for the given download operation
2113 * and then clean up our state (without stop signal).
2115 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
2118 GNUNET_FS_download_signal_suspend_ (void *cls)
2120 struct GNUNET_FS_DownloadContext *dc = cls;
2121 struct GNUNET_FS_ProgressInfo pi;
2123 if (dc->top != NULL)
2124 GNUNET_FS_end_top (dc->h, dc->top);
2125 while (NULL != dc->child_head)
2126 GNUNET_FS_download_signal_suspend_ (dc->child_head);
2127 if (dc->search != NULL)
2129 dc->search->download = NULL;
2132 if (dc->job_queue != NULL)
2134 GNUNET_FS_dequeue_ (dc->job_queue);
2135 dc->job_queue = NULL;
2137 if (dc->parent != NULL)
2138 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2139 dc->parent->child_tail,
2141 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2143 GNUNET_SCHEDULER_cancel (dc->task);
2144 dc->task = GNUNET_SCHEDULER_NO_TASK;
2146 pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
2147 GNUNET_FS_download_make_status_ (&pi, dc);
2150 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2153 if (dc->rfh != NULL)
2155 GNUNET_DISK_file_close (dc->rfh);
2158 GNUNET_FS_free_download_request_ (dc->top_request);
2159 if (dc->active != NULL)
2161 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2164 GNUNET_free_non_null (dc->filename);
2165 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2166 GNUNET_FS_uri_destroy (dc->uri);
2167 GNUNET_free_non_null (dc->temp_filename);
2168 GNUNET_free_non_null (dc->serialization);
2174 * Download parts of a file. Note that this will store
2175 * the blocks at the respective offset in the given file. Also, the
2176 * download is still using the blocking of the underlying FS
2177 * encoding. As a result, the download may *write* outside of the
2178 * given boundaries (if offset and length do not match the 32k FS
2179 * block boundaries). <p>
2181 * This function should be used to focus a download towards a
2182 * particular portion of the file (optimization), not to strictly
2183 * limit the download to exactly those bytes.
2185 * @param h handle to the file sharing subsystem
2186 * @param uri the URI of the file (determines what to download); CHK or LOC URI
2187 * @param meta known metadata for the file (can be NULL)
2188 * @param filename where to store the file, maybe NULL (then no file is
2189 * created on disk and data must be grabbed from the callbacks)
2190 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2191 * can be NULL (in which case we will pick a name if needed); the temporary file
2192 * may already exist, in which case we will try to use the data that is there and
2193 * if it is not what is desired, will overwrite it
2194 * @param offset at what offset should we start the download (typically 0)
2195 * @param length how many bytes should be downloaded starting at offset
2196 * @param anonymity anonymity level to use for the download
2197 * @param options various options
2198 * @param cctx initial value for the client context for this download
2199 * @param parent parent download to associate this download with (use NULL
2200 * for top-level downloads; useful for manually-triggered recursive downloads)
2201 * @return context that can be used to control this download
2203 struct GNUNET_FS_DownloadContext *
2204 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
2205 const struct GNUNET_FS_Uri *uri,
2206 const struct GNUNET_CONTAINER_MetaData *meta,
2207 const char *filename,
2208 const char *tempname,
2212 enum GNUNET_FS_DownloadOptions options,
2214 struct GNUNET_FS_DownloadContext *parent)
2216 struct GNUNET_FS_DownloadContext *dc;
2218 GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
2219 GNUNET_FS_uri_test_loc (uri) );
2221 if ( (offset + length < offset) ||
2222 (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
2228 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2229 "Starting download `%s' of %llu bytes\n",
2231 (unsigned long long) length);
2233 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2235 dc->parent = parent;
2238 GNUNET_CONTAINER_DLL_insert (parent->child_head,
2242 dc->uri = GNUNET_FS_uri_dup (uri);
2243 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
2244 dc->client_info = cctx;
2245 dc->start_time = GNUNET_TIME_absolute_get ();
2246 if (NULL != filename)
2248 dc->filename = GNUNET_strdup (filename);
2249 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2250 GNUNET_DISK_file_size (filename,
2254 if (GNUNET_FS_uri_test_loc (dc->uri))
2255 GNUNET_assert (GNUNET_OK ==
2256 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2258 dc->offset = offset;
2259 dc->length = length;
2260 dc->anonymity = anonymity;
2261 dc->options = options;
2262 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2263 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
2264 if ( (filename == NULL) &&
2265 (is_recursive_download (dc) ) )
2267 if (tempname != NULL)
2268 dc->temp_filename = GNUNET_strdup (tempname);
2270 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2274 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2275 "Download tree has depth %u\n",
2280 dc->top = GNUNET_FS_make_top (dc->h,
2281 &GNUNET_FS_download_signal_suspend_,
2284 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2291 * Download parts of a file based on a search result. The download
2292 * will be associated with the search result (and the association
2293 * will be preserved when serializing/deserializing the state).
2294 * If the search is stopped, the download will not be aborted but
2295 * be 'promoted' to a stand-alone download.
2297 * As with the other download function, this will store
2298 * the blocks at the respective offset in the given file. Also, the
2299 * download is still using the blocking of the underlying FS
2300 * encoding. As a result, the download may *write* outside of the
2301 * given boundaries (if offset and length do not match the 32k FS
2302 * block boundaries). <p>
2304 * The given range can be used to focus a download towards a
2305 * particular portion of the file (optimization), not to strictly
2306 * limit the download to exactly those bytes.
2308 * @param h handle to the file sharing subsystem
2309 * @param sr the search result to use for the download (determines uri and
2310 * meta data and associations)
2311 * @param filename where to store the file, maybe NULL (then no file is
2312 * created on disk and data must be grabbed from the callbacks)
2313 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2314 * can be NULL (in which case we will pick a name if needed); the temporary file
2315 * may already exist, in which case we will try to use the data that is there and
2316 * if it is not what is desired, will overwrite it
2317 * @param offset at what offset should we start the download (typically 0)
2318 * @param length how many bytes should be downloaded starting at offset
2319 * @param anonymity anonymity level to use for the download
2320 * @param options various download options
2321 * @param cctx initial value for the client context for this download
2322 * @return context that can be used to control this download
2324 struct GNUNET_FS_DownloadContext *
2325 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
2326 struct GNUNET_FS_SearchResult *sr,
2327 const char *filename,
2328 const char *tempname,
2332 enum GNUNET_FS_DownloadOptions options,
2335 struct GNUNET_FS_DownloadContext *dc;
2337 if ( (sr == NULL) ||
2338 (sr->download != NULL) )
2343 GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri) ||
2344 GNUNET_FS_uri_test_loc (sr->uri) );
2345 if ( (offset + length < offset) ||
2346 (offset + length > sr->uri->data.chk.file_length) )
2352 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2353 "Starting download `%s' of %llu bytes\n",
2355 (unsigned long long) length);
2357 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2361 if (sr->probe_ctx != NULL)
2363 GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
2364 sr->probe_ctx = NULL;
2366 dc->uri = GNUNET_FS_uri_dup (sr->uri);
2367 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
2368 dc->client_info = cctx;
2369 dc->start_time = GNUNET_TIME_absolute_get ();
2370 if (NULL != filename)
2372 dc->filename = GNUNET_strdup (filename);
2373 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2374 GNUNET_DISK_file_size (filename,
2378 if (GNUNET_FS_uri_test_loc (dc->uri))
2379 GNUNET_assert (GNUNET_OK ==
2380 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2382 dc->offset = offset;
2383 dc->length = length;
2384 dc->anonymity = anonymity;
2385 dc->options = options;
2386 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2387 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
2388 if ( (filename == NULL) &&
2389 (is_recursive_download (dc) ) )
2391 if (tempname != NULL)
2392 dc->temp_filename = GNUNET_strdup (tempname);
2394 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2398 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2399 "Download tree has depth %u\n",
2402 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2409 * Start the downloading process (by entering the queue).
2411 * @param dc our download context
2414 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
2416 if (dc->completed == dc->length)
2418 GNUNET_assert (dc->job_queue == NULL);
2419 dc->job_queue = GNUNET_FS_queue_ (dc->h,
2420 &activate_fs_download,
2421 &deactivate_fs_download,
2423 (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
2428 * Stop a download (aborts if download is incomplete).
2430 * @param dc handle for the download
2431 * @param do_delete delete files of incomplete downloads
2434 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
2437 struct GNUNET_FS_ProgressInfo pi;
2440 if (dc->top != NULL)
2441 GNUNET_FS_end_top (dc->h, dc->top);
2444 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2446 GNUNET_SCHEDULER_cancel (dc->task);
2447 dc->task = GNUNET_SCHEDULER_NO_TASK;
2449 if (dc->search != NULL)
2451 dc->search->download = NULL;
2454 if (dc->job_queue != NULL)
2456 GNUNET_FS_dequeue_ (dc->job_queue);
2457 dc->job_queue = NULL;
2461 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2464 have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
2465 while (NULL != dc->child_head)
2466 GNUNET_FS_download_stop (dc->child_head,
2468 if (dc->parent != NULL)
2469 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2470 dc->parent->child_tail,
2472 if (dc->serialization != NULL)
2473 GNUNET_FS_remove_sync_file_ (dc->h,
2474 ( (dc->parent != NULL) || (dc->search != NULL) )
2475 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2476 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
2478 if ( (GNUNET_YES == have_children) &&
2479 (dc->parent == NULL) )
2480 GNUNET_FS_remove_sync_dir_ (dc->h,
2481 (dc->search != NULL)
2482 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2483 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
2485 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
2486 GNUNET_FS_download_make_status_ (&pi, dc);
2487 GNUNET_FS_free_download_request_ (dc->top_request);
2488 dc->top_request = NULL;
2489 if (dc->active != NULL)
2491 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2494 if (dc->filename != NULL)
2496 if ( (dc->completed != dc->length) &&
2497 (GNUNET_YES == do_delete) )
2499 if (0 != UNLINK (dc->filename))
2500 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
2504 GNUNET_free (dc->filename);
2506 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2507 GNUNET_FS_uri_destroy (dc->uri);
2508 if (NULL != dc->temp_filename)
2510 if (0 != UNLINK (dc->temp_filename))
2511 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
2514 GNUNET_free (dc->temp_filename);
2516 GNUNET_free_non_null (dc->serialization);
2520 /* end of fs_download.c */