2 This file is part of GNUnet.
3 (C) 2001-2011 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - different priority for scheduling probe downloads?
29 #include "gnunet_constants.h"
30 #include "gnunet_fs_service.h"
34 #define DEBUG_DOWNLOAD GNUNET_NO
37 * Determine if the given download (options and meta data) should cause
38 * use to try to do a recursive download.
41 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
43 return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
44 ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
45 ( (dc->meta == NULL) &&
46 ( (NULL == dc->filename) ||
47 ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
49 strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
50 GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
55 * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
56 * only have to truncate the file once we're done).
58 * Given the offset of a block (with respect to the DBLOCKS) and its
59 * depth, return the offset where we would store this block in the
62 * @param fsize overall file size
63 * @param off offset of the block in the file
64 * @param depth depth of the block in the tree, 0 for DBLOCK
65 * @return off for DBLOCKS (depth == treedepth),
66 * otherwise an offset past the end
67 * of the file that does not overlap
68 * with the range for any other block
71 compute_disk_offset (uint64_t fsize,
76 uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
77 uint64_t loff; /* where do IBlocks for depth "i" start? */
78 unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
82 /* first IBlocks start at the end of file, rounded up
83 to full DBLOCK_SIZE */
84 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
85 lsize = ( (fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
86 GNUNET_assert (0 == (off % DBLOCK_SIZE));
87 ioff = (off / DBLOCK_SIZE);
91 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
92 GNUNET_assert (lsize > 0);
93 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
94 ioff /= CHK_PER_INODE;
96 return loff + ioff * sizeof (struct ContentHashKey);
101 * Fill in all of the generic fields for a download event and call the
104 * @param pi structure to fill in
105 * @param dc overall download context
108 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
109 struct GNUNET_FS_DownloadContext *dc)
111 pi->value.download.dc = dc;
112 pi->value.download.cctx
114 pi->value.download.pctx
115 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
116 pi->value.download.sctx
117 = (dc->search == NULL) ? NULL : dc->search->client_info;
118 pi->value.download.uri
120 pi->value.download.filename
122 pi->value.download.size
124 pi->value.download.duration
125 = GNUNET_TIME_absolute_get_duration (dc->start_time);
126 pi->value.download.completed
128 pi->value.download.anonymity
130 pi->value.download.eta
131 = GNUNET_TIME_calculate_eta (dc->start_time,
134 pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
135 if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
136 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
139 dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
145 * We're ready to transmit a search request to the
146 * file-sharing service. Do it. If there is
147 * more than one request pending, try to send
148 * multiple or request another transmission.
151 * @param size number of bytes available in buf
152 * @param buf where the callee should write the message
153 * @return number of bytes written to buf
156 transmit_download_request (void *cls,
162 * Closure for iterator processing results.
164 struct ProcessResultClosure
170 GNUNET_HashCode query;
173 * Data found in P2P network.
178 * Our download context.
180 struct GNUNET_FS_DownloadContext *dc;
183 * Number of bytes in data.
190 enum GNUNET_BLOCK_Type type;
193 * Flag to indicate if this block should be stored on disk.
201 * Iterator over entries in the pending requests in the 'active' map for the
202 * reply that we just got.
204 * @param cls closure (our 'struct ProcessResultClosure')
205 * @param key query for the given value / request
206 * @param value value in the hash map (a 'struct DownloadRequest')
207 * @return GNUNET_YES (we should continue to iterate); unless serious error
210 process_result_with_request (void *cls,
211 const GNUNET_HashCode * key,
216 * We've found a matching block without downloading it.
217 * Encrypt it and pass it to our "receive" function as
218 * if we had received it from the network.
220 * @param dc download in question
221 * @param chk request this relates to
222 * @param dr request details
223 * @param block plaintext data matching request
224 * @param len number of bytes in block
225 * @param do_store should we still store the block on disk?
226 * @return GNUNET_OK on success
229 encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc,
230 const struct ContentHashKey *chk,
231 struct DownloadRequest *dr,
236 struct ProcessResultClosure prc;
238 struct GNUNET_CRYPTO_AesSessionKey sk;
239 struct GNUNET_CRYPTO_AesInitializationVector iv;
240 GNUNET_HashCode query;
242 GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
243 if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
249 return GNUNET_SYSERR;
251 GNUNET_CRYPTO_hash (enc, len, &query);
252 if (0 != memcmp (&query,
254 sizeof (GNUNET_HashCode)))
257 return GNUNET_SYSERR;
260 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
261 "Matching block for `%s' at offset %llu already present, no need for download!\n",
263 (unsigned long long) dr->offset);
265 /* already got it! */
269 prc.type = (0 == dr->depth)
270 ? GNUNET_BLOCK_TYPE_FS_DBLOCK
271 : GNUNET_BLOCK_TYPE_FS_IBLOCK;
272 prc.query = chk->query;
273 prc.do_store = do_store;
274 process_result_with_request (&prc,
282 * We've lost our connection with the FS service.
283 * Re-establish it and re-transmit all of our
286 * @param dc download context that is having trouble
289 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
293 * We found an entry in a directory. Check if the respective child
294 * already exists and if not create the respective child download.
296 * @param cls the parent download
297 * @param filename name of the file in the directory
298 * @param uri URI of the file (CHK or LOC)
299 * @param meta meta data of the file
300 * @param length number of bytes in data
301 * @param data contents of the file (or NULL if they were not inlined)
304 trigger_recursive_download (void *cls,
305 const char *filename,
306 const struct GNUNET_FS_Uri *uri,
307 const struct GNUNET_CONTAINER_MetaData *meta,
313 * We're done downloading a directory. Open the file and
314 * trigger all of the (remaining) child downloads.
316 * @param dc context of download that just completed
319 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
324 struct GNUNET_DISK_FileHandle *h;
325 struct GNUNET_DISK_MapHandle *m;
327 size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
328 size = (size_t) size64;
329 if (size64 != (uint64_t) size)
331 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
332 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
335 if (dc->filename != NULL)
337 h = GNUNET_DISK_file_open (dc->filename,
338 GNUNET_DISK_OPEN_READ,
339 GNUNET_DISK_PERM_NONE);
343 GNUNET_assert (dc->temp_filename != NULL);
344 h = GNUNET_DISK_file_open (dc->temp_filename,
345 GNUNET_DISK_OPEN_READ,
346 GNUNET_DISK_PERM_NONE);
350 data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
353 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
354 _("Directory too large for system address space\n"));
358 GNUNET_FS_directory_list_contents (size,
361 &trigger_recursive_download,
363 GNUNET_DISK_file_unmap (m);
365 GNUNET_DISK_file_close (h);
366 if (dc->filename == NULL)
368 if (0 != UNLINK (dc->temp_filename))
369 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
372 GNUNET_free (dc->temp_filename);
373 dc->temp_filename = NULL;
379 * Check if all child-downloads have completed (or trigger them if
380 * necessary) and once we're completely done, signal completion (and
381 * possibly recurse to parent). This function MUST be called when the
382 * download of a file itself is done or when the download of a file is
383 * done and then later a direct child download has completed (and
384 * hence this download may complete itself).
386 * @param dc download to check for completion of children
389 check_completed (struct GNUNET_FS_DownloadContext *dc)
391 struct GNUNET_FS_ProgressInfo pi;
392 struct GNUNET_FS_DownloadContext *pos;
394 /* first, check if we need to download children */
395 if ( (dc->child_head == NULL) &&
396 (is_recursive_download (dc)) )
397 full_recursive_download (dc);
398 /* then, check if children are done already */
399 pos = dc->child_head;
402 if ( (pos->emsg == NULL) &&
403 (pos->completed < pos->length) )
404 return; /* not done yet */
405 if ( (pos->child_head != NULL) &&
406 (pos->has_finished != GNUNET_YES) )
407 return; /* not transitively done yet */
410 /* All of our children are done, so mark this download done */
411 dc->has_finished = GNUNET_YES;
412 if (dc->job_queue != NULL)
414 GNUNET_FS_dequeue_ (dc->job_queue);
415 dc->job_queue = NULL;
417 GNUNET_FS_download_sync_ (dc);
419 /* signal completion */
420 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
421 GNUNET_FS_download_make_status_ (&pi, dc);
423 /* let parent know */
424 if (dc->parent != NULL)
425 check_completed (dc->parent);
430 * We got a block of plaintext data (from the meta data).
431 * Try it for upward reconstruction of the data. On success,
432 * the top-level block will move to state BRS_DOWNLOAD_UP.
434 * @param dc context for the download
435 * @param dr download request to match against
436 * @param data plaintext data, starting from the beginning of the file
437 * @param data_len number of bytes in data
440 try_match_block (struct GNUNET_FS_DownloadContext *dc,
441 struct DownloadRequest *dr,
445 struct GNUNET_FS_ProgressInfo pi;
447 char enc[DBLOCK_SIZE];
448 struct ContentHashKey chks[CHK_PER_INODE];
449 struct ContentHashKey in_chk;
450 struct GNUNET_CRYPTO_AesSessionKey sk;
451 struct GNUNET_CRYPTO_AesInitializationVector iv;
453 struct DownloadRequest *drc;
454 struct GNUNET_DISK_FileHandle *fh;
461 odata_len = data_len;
462 if (BRS_DOWNLOAD_UP == dr->state)
466 complete = GNUNET_YES;
467 for (i=0;i<dr->num_children;i++)
469 drc = dr->children[i];
473 if (drc->state != BRS_RECONSTRUCT_META_UP)
474 complete = GNUNET_NO;
478 if (GNUNET_YES != complete)
480 data = (const char*) chks;
481 dlen = dr->num_children * sizeof (struct ContentHashKey);
485 if (dr->offset > data_len)
487 dlen = GNUNET_MIN (data_len - dr->offset,
490 GNUNET_CRYPTO_hash (&data[dr->offset],
493 GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
494 if (-1 == GNUNET_CRYPTO_aes_encrypt (&data[dr->offset], dlen,
502 GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
507 dr->state = BRS_RECONSTRUCT_META_UP;
510 if (0 != memcmp (&in_chk,
512 sizeof (struct ContentHashKey)))
514 /* other peer provided bogus meta data */
518 /* write block to disk */
519 fn = dc->filename != NULL
522 fh = GNUNET_DISK_file_open (fn,
523 GNUNET_DISK_OPEN_READWRITE |
524 GNUNET_DISK_OPEN_CREATE |
525 GNUNET_DISK_OPEN_TRUNCATE,
526 GNUNET_DISK_PERM_USER_READ |
527 GNUNET_DISK_PERM_USER_WRITE |
528 GNUNET_DISK_PERM_GROUP_READ |
529 GNUNET_DISK_PERM_OTHER_READ);
532 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
535 GNUNET_asprintf (&dc->emsg,
536 _("Failed to open file `%s' for writing"),
538 GNUNET_DISK_file_close (fh);
539 dr->state = BRS_ERROR;
540 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
541 pi.value.download.specifics.error.message = dc->emsg;
542 GNUNET_FS_download_make_status_ (&pi, dc);
546 GNUNET_DISK_file_write (fh,
550 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
553 GNUNET_asprintf (&dc->emsg,
554 _("Failed to open file `%s' for writing"),
556 GNUNET_DISK_file_close (fh);
557 dr->state = BRS_ERROR;
558 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
559 pi.value.download.specifics.error.message = dc->emsg;
560 GNUNET_FS_download_make_status_ (&pi, dc);
563 GNUNET_DISK_file_close (fh);
565 dr->state = BRS_DOWNLOAD_UP;
566 dc->completed = dc->length;
567 GNUNET_FS_download_sync_ (dc);
568 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
569 pi.value.download.specifics.progress.data = data;
570 pi.value.download.specifics.progress.offset = 0;
571 pi.value.download.specifics.progress.data_len = dlen;
572 pi.value.download.specifics.progress.depth = 0;
573 GNUNET_FS_download_make_status_ (&pi, dc);
574 if ( (NULL != dc->filename) &&
575 (0 != truncate (dc->filename,
576 GNUNET_ntohll (dc->uri->data.chk.file_length))) )
577 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
580 check_completed (dc);
583 /* how did we get here? */
591 * Type of a function that libextractor calls for each
592 * meta data item found. If we find full data meta data,
593 * call 'try_match_block' on it.
595 * @param cls our 'struct GNUNET_FS_DownloadContext*'
596 * @param plugin_name name of the plugin that produced this value;
597 * special values can be used (i.e. '<zlib>' for zlib being
598 * used in the main libextractor library and yielding
600 * @param type libextractor-type describing the meta data
601 * @param format basic format information about data
602 * @param data_mime_type mime-type of data (not of the original file);
603 * can be NULL (if mime-type is not known)
604 * @param data actual meta-data found
605 * @param data_len number of bytes in data
606 * @return 0 to continue extracting, 1 to abort
609 match_full_data (void *cls,
610 const char *plugin_name,
611 enum EXTRACTOR_MetaType type,
612 enum EXTRACTOR_MetaFormat format,
613 const char *data_mime_type,
617 struct GNUNET_FS_DownloadContext *dc = cls;
619 if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
622 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
623 "Found %u bytes of FD!\n",
624 (unsigned int) data_len);
626 if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
629 return 1; /* bogus meta data */
640 * Set the state of the given download request to
641 * BRS_DOWNLOAD_UP and propagate it up the tree.
643 * @param dr download request that is done
646 propagate_up (struct DownloadRequest *dr)
652 dr->state = BRS_DOWNLOAD_UP;
656 for (i=0;i<dr->num_children;i++)
657 if (dr->children[i]->state != BRS_DOWNLOAD_UP)
660 while (i == dr->num_children);
665 * Try top-down reconstruction. Before, the given request node
666 * must have the state BRS_CHK_SET. Afterwards, more nodes may
667 * have that state or advanced to BRS_DOWNLOAD_DOWN or even
668 * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the
671 * @param dc overall download this block belongs to
672 * @param dr block to reconstruct
675 try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
676 struct DownloadRequest *dr)
679 char block[DBLOCK_SIZE];
684 unsigned int chk_off;
685 struct DownloadRequest *drc;
686 uint64_t child_block_size;
687 const struct ContentHashKey *chks;
690 GNUNET_assert (dc->rfh != NULL);
691 GNUNET_assert (dr->state == BRS_CHK_SET);
692 total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
693 GNUNET_assert (dr->depth < dc->treedepth);
694 len = GNUNET_FS_tree_calculate_block_size (total,
697 GNUNET_assert (len <= DBLOCK_SIZE);
698 off = compute_disk_offset (total,
701 if (dc->old_file_size < off + len)
702 return; /* failure */
704 GNUNET_DISK_file_seek (dc->rfh,
706 GNUNET_DISK_SEEK_SET) )
708 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
711 return; /* failure */
714 GNUNET_DISK_file_read (dc->rfh,
718 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
721 return; /* failure */
723 GNUNET_CRYPTO_hash (block, len, &key);
724 if (0 != memcmp (&key,
726 sizeof (GNUNET_HashCode)))
727 return; /* mismatch */
729 encrypt_existing_match (dc,
736 /* hash matches but encrypted block does not, really bad */
737 dr->state = BRS_ERROR;
739 while (dr->parent != NULL)
742 dr->state = BRS_ERROR;
747 dr->state = BRS_DOWNLOAD_DOWN;
749 /* set CHKs for children */
750 up_done = GNUNET_YES;
751 chks = (const struct ContentHashKey*) block;
752 for (i=0;i<dr->num_children;i++)
754 drc = dr->children[i];
755 GNUNET_assert (drc->offset >= dr->offset);
756 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
757 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
758 chk_off = (drc->offset - dr->offset) / child_block_size;
759 if (drc->state == BRS_INIT)
761 drc->state = BRS_CHK_SET;
762 drc->chk = chks[chk_off];
763 try_top_down_reconstruction (dc, drc);
765 if (drc->state != BRS_DOWNLOAD_UP)
766 up_done = GNUNET_NO; /* children not all done */
768 if (up_done == GNUNET_YES)
769 propagate_up (dr); /* children all done (or no children...) */
774 * Schedule the download of the specified block in the tree.
776 * @param dc overall download this block belongs to
777 * @param dr request to schedule
780 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
781 struct DownloadRequest *dr)
790 case BRS_RECONSTRUCT_DOWN:
793 case BRS_RECONSTRUCT_META_UP:
796 case BRS_RECONSTRUCT_UP:
800 /* normal case, start download */
802 case BRS_DOWNLOAD_DOWN:
803 for (i=0;i<dr->num_children;i++)
804 schedule_block_download (dc, dr->children[i]);
806 case BRS_DOWNLOAD_UP:
814 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
815 "Scheduling download at offset %llu and depth %u for `%s'\n",
816 (unsigned long long) dr->offset,
818 GNUNET_h2s (&dr->chk.query));
821 GNUNET_CONTAINER_multihashmap_contains_value (dc->active,
824 return; /* already active */
825 GNUNET_CONTAINER_multihashmap_put (dc->active,
828 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
829 if (dc->client == NULL)
830 return; /* download not active */
831 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
834 dr->is_pending = GNUNET_YES;
836 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
837 sizeof (struct SearchMessage),
838 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
840 &transmit_download_request,
845 #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
848 * We found an entry in a directory. Check if the respective child
849 * already exists and if not create the respective child download.
851 * @param cls the parent download
852 * @param filename name of the file in the directory
853 * @param uri URI of the file (CHK or LOC)
854 * @param meta meta data of the file
855 * @param length number of bytes in data
856 * @param data contents of the file (or NULL if they were not inlined)
859 trigger_recursive_download (void *cls,
860 const char *filename,
861 const struct GNUNET_FS_Uri *uri,
862 const struct GNUNET_CONTAINER_MetaData *meta,
866 struct GNUNET_FS_DownloadContext *dc = cls;
867 struct GNUNET_FS_DownloadContext *cpos;
878 return; /* entry for the directory itself */
879 cpos = dc->child_head;
882 if ( (GNUNET_FS_uri_test_equal (uri,
884 ( (filename != NULL) &&
885 (0 == strcmp (cpos->filename,
891 return; /* already exists */
893 if (NULL == filename)
895 fn = GNUNET_FS_meta_data_suggest_filename (meta);
898 us = GNUNET_FS_uri_to_string (uri);
899 fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_CHK_PREFIX)]);
902 else if (fn[0] == '.')
905 us = GNUNET_FS_uri_to_string (uri);
906 GNUNET_asprintf (&fn,
908 &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], ext);
912 /* change '\' to '/' (this should have happened
913 during insertion, but malicious peers may
914 not have done this) */
915 while (NULL != (pos = strstr (fn, "\\")))
917 /* remove '../' everywhere (again, well-behaved
918 peers don't do this, but don't trust that
919 we did not get something nasty) */
920 while (NULL != (pos = strstr (fn, "../")))
928 if (dc->filename == NULL)
934 dn = GNUNET_strdup (dc->filename);
935 GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
937 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
938 GNUNET_FS_DIRECTORY_EXT)) );
939 sfn = GNUNET_strdup (filename);
940 while ( (strlen (sfn) > 0) &&
941 (filename[strlen(sfn)-1] == '/') )
942 sfn[strlen(sfn)-1] = '\0';
943 if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
945 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
946 GNUNET_FS_DIRECTORY_EXT)) )
947 dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
948 if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
949 ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
951 strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
952 GNUNET_FS_DIRECTORY_EXT)) ) )
954 GNUNET_asprintf (&full_name,
959 GNUNET_FS_DIRECTORY_EXT);
963 GNUNET_asprintf (&full_name,
972 if ( (full_name != NULL) &&
974 GNUNET_DISK_directory_create_for_file (full_name)) )
976 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
977 _("Failed to create directory for recursive download of `%s'\n"),
979 GNUNET_free (full_name);
980 GNUNET_free_non_null (fn);
986 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
987 "Triggering recursive download of size %llu with %u bytes MD\n",
988 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
989 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (meta));
991 GNUNET_FS_download_start (dc->h,
994 full_name, temp_name,
996 GNUNET_FS_uri_chk_get_file_size (uri),
1001 GNUNET_free_non_null (full_name);
1002 GNUNET_free_non_null (temp_name);
1003 GNUNET_free_non_null (fn);
1008 * (recursively) free download request structure
1010 * @param dr request to free
1013 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
1019 for (i=0;i<dr->num_children;i++)
1020 GNUNET_FS_free_download_request_ (dr->children[i]);
1021 GNUNET_free_non_null (dr->children);
1027 * Iterator over entries in the pending requests in the 'active' map for the
1028 * reply that we just got.
1030 * @param cls closure (our 'struct ProcessResultClosure')
1031 * @param key query for the given value / request
1032 * @param value value in the hash map (a 'struct DownloadRequest')
1033 * @return GNUNET_YES (we should continue to iterate); unless serious error
1036 process_result_with_request (void *cls,
1037 const GNUNET_HashCode *key,
1040 struct ProcessResultClosure *prc = cls;
1041 struct DownloadRequest *dr = value;
1042 struct GNUNET_FS_DownloadContext *dc = prc->dc;
1043 struct DownloadRequest *drc;
1044 struct GNUNET_DISK_FileHandle *fh = NULL;
1045 struct GNUNET_CRYPTO_AesSessionKey skey;
1046 struct GNUNET_CRYPTO_AesInitializationVector iv;
1048 struct GNUNET_FS_ProgressInfo pi;
1053 struct ContentHashKey *chkarr;
1056 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1057 "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n",
1060 (unsigned long long) dr->offset,
1061 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1064 bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
1067 if (prc->size != bs)
1069 GNUNET_asprintf (&dc->emsg,
1070 _("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"),
1073 (unsigned long long) dr->offset,
1074 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
1076 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1079 while (dr->parent != NULL)
1081 dr->state = BRS_ERROR;
1084 dr->state = BRS_ERROR;
1088 (void) GNUNET_CONTAINER_multihashmap_remove (dc->active,
1091 if (GNUNET_YES == dr->is_pending)
1093 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1096 dr->is_pending = GNUNET_NO;
1100 GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
1101 if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
1108 dc->emsg = GNUNET_strdup (_("internal error decrypting content"));
1111 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
1115 if ( ( GNUNET_YES == prc->do_store) &&
1116 ( (dc->filename != NULL) ||
1117 (is_recursive_download (dc)) ) &&
1118 ( (dr->depth == dc->treedepth) ||
1119 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
1121 fh = GNUNET_DISK_file_open (dc->filename != NULL
1123 : dc->temp_filename,
1124 GNUNET_DISK_OPEN_READWRITE |
1125 GNUNET_DISK_OPEN_CREATE,
1126 GNUNET_DISK_PERM_USER_READ |
1127 GNUNET_DISK_PERM_USER_WRITE |
1128 GNUNET_DISK_PERM_GROUP_READ |
1129 GNUNET_DISK_PERM_OTHER_READ);
1132 GNUNET_asprintf (&dc->emsg,
1133 _("Download failed: could not open file `%s': %s\n"),
1139 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1140 "Saving decrypted block to disk at offset %llu\n",
1141 (unsigned long long) off);
1144 GNUNET_DISK_file_seek (fh,
1146 GNUNET_DISK_SEEK_SET) ) )
1148 GNUNET_asprintf (&dc->emsg,
1149 _("Failed to seek to offset %llu in file `%s': %s\n"),
1150 (unsigned long long) off,
1156 GNUNET_DISK_file_write (fh,
1160 GNUNET_asprintf (&dc->emsg,
1161 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1162 (unsigned int) prc->size,
1163 (unsigned long long) off,
1168 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1174 /* DBLOCK, update progress and try recursion if applicable */
1176 if (dr->offset < dc->offset)
1178 /* starting offset begins in the middle of pt,
1179 do not count first bytes as progress */
1180 GNUNET_assert (app > (dc->offset - dr->offset));
1181 app -= (dc->offset - dr->offset);
1183 if (dr->offset + prc->size > dc->offset + dc->length)
1185 /* end of block is after relevant range,
1186 do not count last bytes as progress */
1187 GNUNET_assert (app > (dr->offset + prc->size) - (dc->offset + dc->length));
1188 app -= (dr->offset + prc->size) - (dc->offset + dc->length);
1190 dc->completed += app;
1192 /* do recursive download if option is set and either meta data
1193 says it is a directory or if no meta data is given AND filename
1194 ends in '.gnd' (top-level case) */
1195 if (is_recursive_download (dc))
1196 GNUNET_FS_directory_list_contents (prc->size,
1199 &trigger_recursive_download,
1203 dr->state = BRS_DOWNLOAD_DOWN;
1204 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1205 pi.value.download.specifics.progress.data = pt;
1206 pi.value.download.specifics.progress.offset = dr->offset;
1207 pi.value.download.specifics.progress.data_len = prc->size;
1208 pi.value.download.specifics.progress.depth = dr->depth;
1209 GNUNET_FS_download_make_status_ (&pi, dc);
1210 GNUNET_assert (dc->completed <= dc->length);
1214 if (dc->completed == dc->length)
1216 /* download completed, signal */
1218 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1219 "Download completed, truncating file to desired length %llu\n",
1220 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1222 /* truncate file to size (since we store IBlocks at the end) */
1223 if (dc->filename != NULL)
1225 if (0 != truncate (dc->filename,
1226 GNUNET_ntohll (dc->uri->data.chk.file_length)))
1227 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1231 GNUNET_assert (dr->depth == 0);
1232 check_completed (dc);
1236 /* bottom of the tree, no child downloads possible, just sync */
1237 GNUNET_FS_download_sync_ (dc);
1242 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1243 "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1245 (unsigned long long) dr->offset);
1247 GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1248 chkarr = (struct ContentHashKey*) pt;
1249 for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1251 drc = dr->children[i];
1255 drc->chk = chkarr[i];
1256 drc->state = BRS_CHK_SET;
1257 schedule_block_download (dc, drc);
1259 case BRS_RECONSTRUCT_DOWN:
1262 case BRS_RECONSTRUCT_META_UP:
1265 case BRS_RECONSTRUCT_UP:
1271 case BRS_DOWNLOAD_DOWN:
1274 case BRS_DOWNLOAD_UP:
1285 GNUNET_FS_download_sync_ (dc);
1290 GNUNET_DISK_file_close (fh);
1291 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1292 pi.value.download.specifics.error.message = dc->emsg;
1293 GNUNET_FS_download_make_status_ (&pi, dc);
1294 /* abort all pending requests */
1297 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1300 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1301 dc->in_receive = GNUNET_NO;
1303 GNUNET_FS_free_download_request_ (dc->top_request);
1304 dc->top_request = NULL;
1305 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1307 dc->pending_head = NULL;
1308 dc->pending_tail = NULL;
1309 GNUNET_FS_download_sync_ (dc);
1315 * Process a download result.
1317 * @param dc our download context
1318 * @param type type of the result
1319 * @param data the (encrypted) response
1320 * @param size size of data
1323 process_result (struct GNUNET_FS_DownloadContext *dc,
1324 enum GNUNET_BLOCK_Type type,
1328 struct ProcessResultClosure prc;
1334 prc.do_store = GNUNET_YES;
1335 GNUNET_CRYPTO_hash (data, size, &prc.query);
1337 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1338 "Received result for query `%s' from `%s'-service\n",
1339 GNUNET_h2s (&prc.query),
1342 GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1344 &process_result_with_request,
1350 * Type of a function to call when we receive a message
1353 * @param cls closure
1354 * @param msg message received, NULL on timeout or fatal error
1357 receive_results (void *cls,
1358 const struct GNUNET_MessageHeader * msg)
1360 struct GNUNET_FS_DownloadContext *dc = cls;
1361 const struct PutMessage *cm;
1364 if ( (NULL == msg) ||
1365 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1366 (sizeof (struct PutMessage) > ntohs(msg->size)) )
1368 GNUNET_break (msg == NULL);
1372 msize = ntohs(msg->size);
1373 cm = (const struct PutMessage*) msg;
1377 msize - sizeof (struct PutMessage));
1378 if (dc->client == NULL)
1379 return; /* fatal error */
1380 /* continue receiving */
1381 GNUNET_CLIENT_receive (dc->client,
1384 GNUNET_TIME_UNIT_FOREVER_REL);
1390 * We're ready to transmit a search request to the
1391 * file-sharing service. Do it. If there is
1392 * more than one request pending, try to send
1393 * multiple or request another transmission.
1395 * @param cls closure
1396 * @param size number of bytes available in buf
1397 * @param buf where the callee should write the message
1398 * @return number of bytes written to buf
1401 transmit_download_request (void *cls,
1405 struct GNUNET_FS_DownloadContext *dc = cls;
1407 struct SearchMessage *sm;
1408 struct DownloadRequest *dr;
1414 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1415 "Transmitting download request failed, trying to reconnect\n");
1420 GNUNET_assert (size >= sizeof (struct SearchMessage));
1423 while ( (NULL != (dr = dc->pending_head)) &&
1424 (size >= msize + sizeof (struct SearchMessage)) )
1427 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1428 "Transmitting download request for `%s' to `%s'-service\n",
1429 GNUNET_h2s (&dr->chk.query),
1432 memset (sm, 0, sizeof (struct SearchMessage));
1433 sm->header.size = htons (sizeof (struct SearchMessage));
1434 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1435 if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1436 sm->options = htonl (1);
1438 sm->options = htonl (0);
1440 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
1442 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
1443 sm->anonymity_level = htonl (dc->anonymity);
1444 sm->target = dc->target.hashPubKey;
1445 sm->query = dr->chk.query;
1446 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1449 dr->is_pending = GNUNET_NO;
1450 msize += sizeof (struct SearchMessage);
1453 if (dc->pending_head != NULL)
1455 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1456 sizeof (struct SearchMessage),
1457 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1459 &transmit_download_request,
1461 GNUNET_assert (dc->th != NULL);
1463 if (GNUNET_NO == dc->in_receive)
1465 dc->in_receive = GNUNET_YES;
1466 GNUNET_CLIENT_receive (dc->client,
1469 GNUNET_TIME_UNIT_FOREVER_REL);
1476 * Reconnect to the FS service and transmit our queries NOW.
1478 * @param cls our download context
1482 do_reconnect (void *cls,
1483 const struct GNUNET_SCHEDULER_TaskContext *tc)
1485 struct GNUNET_FS_DownloadContext *dc = cls;
1486 struct GNUNET_CLIENT_Connection *client;
1488 dc->task = GNUNET_SCHEDULER_NO_TASK;
1489 client = GNUNET_CLIENT_connect ("fs",
1493 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1494 "Connecting to `%s'-service failed, will try again.\n",
1499 dc->client = client;
1500 if (dc->pending_head != NULL)
1502 dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1503 sizeof (struct SearchMessage),
1504 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1506 &transmit_download_request,
1508 GNUNET_assert (dc->th != NULL);
1514 * Add entries to the pending list.
1516 * @param cls our download context
1518 * @param entry entry of type "struct DownloadRequest"
1522 retry_entry (void *cls,
1523 const GNUNET_HashCode *key,
1526 struct GNUNET_FS_DownloadContext *dc = cls;
1527 struct DownloadRequest *dr = entry;
1531 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
1534 dr->is_pending = GNUNET_YES;
1540 * We've lost our connection with the FS service.
1541 * Re-establish it and re-transmit all of our
1544 * @param dc download context that is having trouble
1547 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1550 if (NULL != dc->client)
1553 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1554 "Moving all requests back to pending list\n");
1558 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1561 /* full reset of the pending list */
1562 dc->pending_head = NULL;
1563 dc->pending_tail = NULL;
1564 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1567 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1568 dc->in_receive = GNUNET_NO;
1572 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1573 "Will try to reconnect in 1s\n");
1576 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1583 * We're allowed to ask the FS service for our blocks. Start the download.
1585 * @param cls the 'struct GNUNET_FS_DownloadContext'
1586 * @param client handle to use for communcation with FS (we must destroy it!)
1589 activate_fs_download (void *cls,
1590 struct GNUNET_CLIENT_Connection *client)
1592 struct GNUNET_FS_DownloadContext *dc = cls;
1593 struct GNUNET_FS_ProgressInfo pi;
1596 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1597 "Download activated\n");
1599 GNUNET_assert (NULL != client);
1600 GNUNET_assert (dc->client == NULL);
1601 GNUNET_assert (dc->th == NULL);
1602 dc->client = client;
1603 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1604 GNUNET_FS_download_make_status_ (&pi, dc);
1605 dc->pending_head = NULL;
1606 dc->pending_tail = NULL;
1607 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1611 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1612 "Asking for transmission to FS service\n");
1614 if (dc->pending_head != NULL)
1616 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1617 sizeof (struct SearchMessage),
1618 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1620 &transmit_download_request,
1622 GNUNET_assert (dc->th != NULL);
1628 * We must stop to ask the FS service for our blocks. Pause the download.
1630 * @param cls the 'struct GNUNET_FS_DownloadContext'
1633 deactivate_fs_download (void *cls)
1635 struct GNUNET_FS_DownloadContext *dc = cls;
1636 struct GNUNET_FS_ProgressInfo pi;
1639 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1640 "Download deactivated\n");
1644 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1647 if (NULL != dc->client)
1649 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1650 dc->in_receive = GNUNET_NO;
1653 dc->pending_head = NULL;
1654 dc->pending_tail = NULL;
1655 pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1656 GNUNET_FS_download_make_status_ (&pi, dc);
1661 * (recursively) Create a download request structure.
1663 * @param parent parent of the current entry
1664 * @param depth depth of the current entry, 0 are the DBLOCKs,
1665 * top level block is 'dc->treedepth - 1'
1666 * @param dr_offset offset in the original file this block maps to
1667 * (as in, offset of the first byte of the first DBLOCK
1668 * in the subtree rooted in the returned download request tree)
1669 * @param file_start_offset desired starting offset for the download
1670 * in the original file; requesting tree should not contain
1671 * DBLOCKs prior to the file_start_offset
1672 * @param desired_length desired number of bytes the user wanted to access
1673 * (from file_start_offset). Resulting tree should not contain
1674 * DBLOCKs after file_start_offset + file_length.
1675 * @return download request tree for the given range of DBLOCKs at
1676 * the specified depth
1678 static struct DownloadRequest *
1679 create_download_request (struct DownloadRequest *parent,
1682 uint64_t file_start_offset,
1683 uint64_t desired_length)
1685 struct DownloadRequest *dr;
1687 unsigned int head_skip;
1688 uint64_t child_block_size;
1690 dr = GNUNET_malloc (sizeof (struct DownloadRequest));
1691 dr->parent = parent;
1693 dr->offset = dr_offset;
1696 child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
1698 /* calculate how many blocks at this level are not interesting
1699 from the start (rounded down), either because of the requested
1700 file offset or because this IBlock is further along */
1701 if (dr_offset < file_start_offset)
1702 head_skip = file_start_offset / child_block_size;
1704 head_skip = dr_offset / child_block_size;
1706 /* calculate index of last block at this level that is interesting (rounded up) */
1707 dr->num_children = file_start_offset + desired_length / child_block_size;
1708 if (dr->num_children * child_block_size < file_start_offset + desired_length)
1709 dr->num_children++; /* round up */
1711 /* now we can get the total number of children for this block */
1712 dr->num_children -= head_skip;
1713 if (dr->num_children > CHK_PER_INODE)
1714 dr->num_children = CHK_PER_INODE; /* cap at max */
1716 /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
1717 GNUNET_assert (dr->num_children > 0);
1719 dr->children = GNUNET_malloc (dr->num_children *
1720 sizeof (struct DownloadRequest *));
1721 for (i=0;i<dr->num_children;i++)
1722 dr->children[i] = create_download_request (dr,
1724 dr_offset + i * child_block_size,
1733 * Continuation after a possible attempt to reconstruct
1734 * the current IBlock from the existing file.
1736 * @param cls the 'struct ReconstructContext'
1737 * @param tc scheduler context
1740 reconstruct_cont (void *cls,
1741 const struct GNUNET_SCHEDULER_TaskContext *tc)
1743 struct GNUNET_FS_DownloadContext *dc = cls;
1745 /* clean up state from tree encoder */
1748 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
1751 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
1753 GNUNET_SCHEDULER_cancel (dc->task);
1754 dc->task = GNUNET_SCHEDULER_NO_TASK;
1756 if (dc->rfh != NULL)
1758 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
1761 /* start "normal" download */
1762 schedule_block_download (dc,
1768 * Task requesting the next block from the tree encoder.
1770 * @param cls the 'struct GNUJNET_FS_DownloadContext' we're processing
1771 * @param tc task context
1774 get_next_block (void *cls,
1775 const struct GNUNET_SCHEDULER_TaskContext *tc)
1777 struct GNUNET_FS_DownloadContext *dc = cls;
1779 dc->task = GNUNET_SCHEDULER_NO_TASK;
1780 GNUNET_FS_tree_encoder_next (dc->te);
1786 * Function called asking for the current (encoded)
1787 * block to be processed. After processing the
1788 * client should either call "GNUNET_FS_tree_encode_next"
1789 * or (on error) "GNUNET_FS_tree_encode_finish".
1791 * This function checks if the content on disk matches
1792 * the expected content based on the URI.
1794 * @param cls closure
1795 * @param chk content hash key for the block
1796 * @param offset offset of the block
1797 * @param depth depth of the block, 0 for DBLOCK
1798 * @param type type of the block (IBLOCK or DBLOCK)
1799 * @param block the (encrypted) block
1800 * @param block_size size of block (in bytes)
1803 reconstruct_cb (void *cls,
1804 const struct ContentHashKey *chk,
1807 enum GNUNET_BLOCK_Type type,
1809 uint16_t block_size)
1811 struct GNUNET_FS_DownloadContext *dc = cls;
1812 struct GNUNET_FS_ProgressInfo pi;
1813 struct DownloadRequest *dr;
1817 /* find corresponding request entry */
1818 dr = dc->top_request;
1819 while (dr->depth > depth)
1821 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1822 chld = (offset - dr->offset) / blen;
1823 GNUNET_assert (chld < dr->num_children);
1824 dr = dr->children[chld];
1830 case BRS_RECONSTRUCT_DOWN:
1832 case BRS_RECONSTRUCT_META_UP:
1834 case BRS_RECONSTRUCT_UP:
1837 if (0 == memcmp (chk,
1839 sizeof (struct ContentHashKey)))
1841 /* block matches, hence tree below matches;
1842 this request is done! */
1843 dr->state = BRS_DOWNLOAD_UP;
1844 /* calculate how many bytes of payload this block
1846 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1847 /* how many of those bytes are in the requested range? */
1848 blen = GNUNET_MIN (blen,
1849 dc->length + dc->offset - dr->offset);
1850 /* signal progress */
1851 dc->completed += blen;
1852 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1853 pi.value.download.specifics.progress.data = NULL;
1854 pi.value.download.specifics.progress.offset = offset;
1855 pi.value.download.specifics.progress.data_len = 0;
1856 pi.value.download.specifics.progress.depth = 0;
1857 GNUNET_FS_download_make_status_ (&pi, dc);
1863 case BRS_DOWNLOAD_DOWN:
1865 case BRS_DOWNLOAD_UP:
1873 if ( (dr == dc->top_request) &&
1874 (dr->state == BRS_DOWNLOAD_UP) )
1876 check_completed (dc);
1879 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
1885 * Function called by the tree encoder to obtain a block of plaintext
1886 * data (for the lowest level of the tree).
1888 * @param cls our 'struct ReconstructContext'
1889 * @param offset identifies which block to get
1890 * @param max (maximum) number of bytes to get; returning
1891 * fewer will also cause errors
1892 * @param buf where to copy the plaintext buffer
1893 * @param emsg location to store an error message (on error)
1894 * @return number of bytes copied to buf, 0 on error
1897 fh_reader (void *cls,
1903 struct GNUNET_FS_DownloadContext *dc = cls;
1904 struct GNUNET_DISK_FileHandle *fh = dc->rfh;
1909 GNUNET_DISK_file_seek (fh,
1911 GNUNET_DISK_SEEK_SET))
1913 *emsg = GNUNET_strdup (strerror (errno));
1916 ret = GNUNET_DISK_file_read (fh, buf, max);
1919 *emsg = GNUNET_strdup (strerror (errno));
1927 * Task that creates the initial (top-level) download
1928 * request for the file.
1930 * @param cls the 'struct GNUNET_FS_DownloadContext'
1931 * @param tc scheduler context
1934 GNUNET_FS_download_start_task_ (void *cls,
1935 const struct GNUNET_SCHEDULER_TaskContext *tc)
1937 struct GNUNET_FS_DownloadContext *dc = cls;
1938 struct GNUNET_FS_ProgressInfo pi;
1939 struct GNUNET_DISK_FileHandle *fh;
1942 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1943 "Start task running...\n");
1945 dc->task = GNUNET_SCHEDULER_NO_TASK;
1946 if (dc->length == 0)
1948 /* no bytes required! */
1949 if (dc->filename != NULL)
1951 fh = GNUNET_DISK_file_open (dc->filename,
1952 GNUNET_DISK_OPEN_READWRITE |
1953 GNUNET_DISK_OPEN_CREATE |
1954 ( (0 == GNUNET_FS_uri_chk_get_file_size (dc->uri))
1955 ? GNUNET_DISK_OPEN_TRUNCATE : 0),
1956 GNUNET_DISK_PERM_USER_READ |
1957 GNUNET_DISK_PERM_USER_WRITE |
1958 GNUNET_DISK_PERM_GROUP_READ |
1959 GNUNET_DISK_PERM_OTHER_READ);
1960 GNUNET_DISK_file_close (fh);
1962 GNUNET_FS_download_sync_ (dc);
1963 check_completed (dc);
1966 if (dc->emsg != NULL)
1968 if (dc->top_request == NULL)
1970 dc->top_request = create_download_request (NULL, dc->treedepth - 1, 0,
1971 dc->offset, dc->length);
1972 dc->top_request->state = BRS_CHK_SET;
1973 dc->top_request->chk = (dc->uri->type == chk)
1974 ? dc->uri->data.chk.chk
1975 : dc->uri->data.loc.fi.chk;
1977 GNUNET_FS_download_sync_ (dc);
1978 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1979 pi.value.download.specifics.start.meta = dc->meta;
1980 GNUNET_FS_download_make_status_ (&pi, dc);
1982 GNUNET_FS_download_start_downloading_ (dc);
1983 /* attempt reconstruction from disk */
1984 if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
1985 dc->rfh = GNUNET_DISK_file_open (dc->filename,
1986 GNUNET_DISK_OPEN_READ,
1987 GNUNET_DISK_PERM_NONE);
1988 if (dc->top_request->state == BRS_CHK_SET)
1990 if (dc->rfh != NULL)
1992 /* first, try top-down */
1994 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1995 "Trying top-down reconstruction for `%s'\n",
1998 try_top_down_reconstruction (dc, dc->top_request);
1999 switch (dc->top_request->state)
2003 case BRS_DOWNLOAD_DOWN:
2004 break; /* normal, some blocks already down */
2005 case BRS_DOWNLOAD_UP:
2006 /* already done entirely, party! */
2007 if (dc->rfh != NULL)
2009 /* avoid hanging on to file handle longer than
2011 GNUNET_DISK_file_close (dc->rfh);
2016 GNUNET_asprintf (&dc->emsg,
2018 GNUNET_FS_download_sync_ (dc);
2019 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
2020 pi.value.download.specifics.error.message = dc->emsg;
2021 GNUNET_FS_download_make_status_ (&pi, dc);
2029 /* attempt reconstruction from meta data */
2030 if ( (GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) &&
2031 (NULL != dc->meta) )
2034 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2035 "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
2036 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
2037 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (dc->meta));
2039 GNUNET_CONTAINER_meta_data_iterate (dc->meta,
2042 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2044 if (dc->rfh != NULL)
2046 /* avoid hanging on to file handle longer than
2048 GNUNET_DISK_file_close (dc->rfh);
2051 return; /* finished, status update was already done for us */
2054 if (dc->rfh != NULL)
2056 /* finally, try bottom-up */
2058 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2059 "Trying bottom-up reconstruction of file `%s'\n",
2062 dc->te = GNUNET_FS_tree_encoder_create (dc->h,
2069 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
2074 /* simple, top-level download */
2075 schedule_block_download (dc,
2078 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2079 check_completed (dc);
2084 * Create SUSPEND event for the given download operation
2085 * and then clean up our state (without stop signal).
2087 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
2090 GNUNET_FS_download_signal_suspend_ (void *cls)
2092 struct GNUNET_FS_DownloadContext *dc = cls;
2093 struct GNUNET_FS_ProgressInfo pi;
2095 if (dc->top != NULL)
2096 GNUNET_FS_end_top (dc->h, dc->top);
2097 while (NULL != dc->child_head)
2098 GNUNET_FS_download_signal_suspend_ (dc->child_head);
2099 if (dc->search != NULL)
2101 dc->search->download = NULL;
2104 if (dc->job_queue != NULL)
2106 GNUNET_FS_dequeue_ (dc->job_queue);
2107 dc->job_queue = NULL;
2109 if (dc->parent != NULL)
2110 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2111 dc->parent->child_tail,
2113 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2115 GNUNET_SCHEDULER_cancel (dc->task);
2116 dc->task = GNUNET_SCHEDULER_NO_TASK;
2118 pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
2119 GNUNET_FS_download_make_status_ (&pi, dc);
2122 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2125 if (dc->rfh != NULL)
2127 GNUNET_DISK_file_close (dc->rfh);
2130 GNUNET_FS_free_download_request_ (dc->top_request);
2131 if (dc->active != NULL)
2133 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2136 GNUNET_free_non_null (dc->filename);
2137 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2138 GNUNET_FS_uri_destroy (dc->uri);
2139 GNUNET_free_non_null (dc->temp_filename);
2140 GNUNET_free_non_null (dc->serialization);
2146 * Download parts of a file. Note that this will store
2147 * the blocks at the respective offset in the given file. Also, the
2148 * download is still using the blocking of the underlying FS
2149 * encoding. As a result, the download may *write* outside of the
2150 * given boundaries (if offset and length do not match the 32k FS
2151 * block boundaries). <p>
2153 * This function should be used to focus a download towards a
2154 * particular portion of the file (optimization), not to strictly
2155 * limit the download to exactly those bytes.
2157 * @param h handle to the file sharing subsystem
2158 * @param uri the URI of the file (determines what to download); CHK or LOC URI
2159 * @param meta known metadata for the file (can be NULL)
2160 * @param filename where to store the file, maybe NULL (then no file is
2161 * created on disk and data must be grabbed from the callbacks)
2162 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2163 * can be NULL (in which case we will pick a name if needed); the temporary file
2164 * may already exist, in which case we will try to use the data that is there and
2165 * if it is not what is desired, will overwrite it
2166 * @param offset at what offset should we start the download (typically 0)
2167 * @param length how many bytes should be downloaded starting at offset
2168 * @param anonymity anonymity level to use for the download
2169 * @param options various options
2170 * @param cctx initial value for the client context for this download
2171 * @param parent parent download to associate this download with (use NULL
2172 * for top-level downloads; useful for manually-triggered recursive downloads)
2173 * @return context that can be used to control this download
2175 struct GNUNET_FS_DownloadContext *
2176 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
2177 const struct GNUNET_FS_Uri *uri,
2178 const struct GNUNET_CONTAINER_MetaData *meta,
2179 const char *filename,
2180 const char *tempname,
2184 enum GNUNET_FS_DownloadOptions options,
2186 struct GNUNET_FS_DownloadContext *parent)
2188 struct GNUNET_FS_DownloadContext *dc;
2190 GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
2191 GNUNET_FS_uri_test_loc (uri) );
2193 if ( (offset + length < offset) ||
2194 (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
2200 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2201 "Starting download `%s' of %llu bytes\n",
2203 (unsigned long long) length);
2205 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2207 dc->parent = parent;
2210 GNUNET_CONTAINER_DLL_insert (parent->child_head,
2214 dc->uri = GNUNET_FS_uri_dup (uri);
2215 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
2216 dc->client_info = cctx;
2217 dc->start_time = GNUNET_TIME_absolute_get ();
2218 if (NULL != filename)
2220 dc->filename = GNUNET_strdup (filename);
2221 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2222 GNUNET_DISK_file_size (filename,
2226 if (GNUNET_FS_uri_test_loc (dc->uri))
2227 GNUNET_assert (GNUNET_OK ==
2228 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2230 dc->offset = offset;
2231 dc->length = length;
2232 dc->anonymity = anonymity;
2233 dc->options = options;
2234 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2235 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
2236 if ( (filename == NULL) &&
2237 (is_recursive_download (dc) ) )
2239 if (tempname != NULL)
2240 dc->temp_filename = GNUNET_strdup (tempname);
2242 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2247 "Download tree has depth %u\n",
2252 dc->top = GNUNET_FS_make_top (dc->h,
2253 &GNUNET_FS_download_signal_suspend_,
2256 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2263 * Download parts of a file based on a search result. The download
2264 * will be associated with the search result (and the association
2265 * will be preserved when serializing/deserializing the state).
2266 * If the search is stopped, the download will not be aborted but
2267 * be 'promoted' to a stand-alone download.
2269 * As with the other download function, this will store
2270 * the blocks at the respective offset in the given file. Also, the
2271 * download is still using the blocking of the underlying FS
2272 * encoding. As a result, the download may *write* outside of the
2273 * given boundaries (if offset and length do not match the 32k FS
2274 * block boundaries). <p>
2276 * The given range can be used to focus a download towards a
2277 * particular portion of the file (optimization), not to strictly
2278 * limit the download to exactly those bytes.
2280 * @param h handle to the file sharing subsystem
2281 * @param sr the search result to use for the download (determines uri and
2282 * meta data and associations)
2283 * @param filename where to store the file, maybe NULL (then no file is
2284 * created on disk and data must be grabbed from the callbacks)
2285 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2286 * can be NULL (in which case we will pick a name if needed); the temporary file
2287 * may already exist, in which case we will try to use the data that is there and
2288 * if it is not what is desired, will overwrite it
2289 * @param offset at what offset should we start the download (typically 0)
2290 * @param length how many bytes should be downloaded starting at offset
2291 * @param anonymity anonymity level to use for the download
2292 * @param options various download options
2293 * @param cctx initial value for the client context for this download
2294 * @return context that can be used to control this download
2296 struct GNUNET_FS_DownloadContext *
2297 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
2298 struct GNUNET_FS_SearchResult *sr,
2299 const char *filename,
2300 const char *tempname,
2304 enum GNUNET_FS_DownloadOptions options,
2307 struct GNUNET_FS_DownloadContext *dc;
2309 if ( (sr == NULL) ||
2310 (sr->download != NULL) )
2315 GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri) ||
2316 GNUNET_FS_uri_test_loc (sr->uri) );
2317 if ( (offset + length < offset) ||
2318 (offset + length > sr->uri->data.chk.file_length) )
2324 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2325 "Starting download `%s' of %llu bytes\n",
2327 (unsigned long long) length);
2329 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2333 if (sr->probe_ctx != NULL)
2335 GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
2336 sr->probe_ctx = NULL;
2338 dc->uri = GNUNET_FS_uri_dup (sr->uri);
2339 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
2340 dc->client_info = cctx;
2341 dc->start_time = GNUNET_TIME_absolute_get ();
2342 if (NULL != filename)
2344 dc->filename = GNUNET_strdup (filename);
2345 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2346 GNUNET_DISK_file_size (filename,
2350 if (GNUNET_FS_uri_test_loc (dc->uri))
2351 GNUNET_assert (GNUNET_OK ==
2352 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2354 dc->offset = offset;
2355 dc->length = length;
2356 dc->anonymity = anonymity;
2357 dc->options = options;
2358 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2359 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
2360 if ( (filename == NULL) &&
2361 (is_recursive_download (dc) ) )
2363 if (tempname != NULL)
2364 dc->temp_filename = GNUNET_strdup (tempname);
2366 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2370 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2371 "Download tree has depth %u\n",
2374 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2381 * Start the downloading process (by entering the queue).
2383 * @param dc our download context
2386 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
2388 if (dc->completed == dc->length)
2390 GNUNET_assert (dc->job_queue == NULL);
2391 dc->job_queue = GNUNET_FS_queue_ (dc->h,
2392 &activate_fs_download,
2393 &deactivate_fs_download,
2395 (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
2400 * Stop a download (aborts if download is incomplete).
2402 * @param dc handle for the download
2403 * @param do_delete delete files of incomplete downloads
2406 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
2409 struct GNUNET_FS_ProgressInfo pi;
2412 if (dc->top != NULL)
2413 GNUNET_FS_end_top (dc->h, dc->top);
2416 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2418 GNUNET_SCHEDULER_cancel (dc->task);
2419 dc->task = GNUNET_SCHEDULER_NO_TASK;
2421 if (dc->search != NULL)
2423 dc->search->download = NULL;
2426 if (dc->job_queue != NULL)
2428 GNUNET_FS_dequeue_ (dc->job_queue);
2429 dc->job_queue = NULL;
2433 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2436 have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
2437 while (NULL != dc->child_head)
2438 GNUNET_FS_download_stop (dc->child_head,
2440 if (dc->parent != NULL)
2441 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2442 dc->parent->child_tail,
2444 if (dc->serialization != NULL)
2445 GNUNET_FS_remove_sync_file_ (dc->h,
2446 ( (dc->parent != NULL) || (dc->search != NULL) )
2447 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2448 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
2450 if ( (GNUNET_YES == have_children) &&
2451 (dc->parent == NULL) )
2452 GNUNET_FS_remove_sync_dir_ (dc->h,
2453 (dc->search != NULL)
2454 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2455 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
2457 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
2458 GNUNET_FS_download_make_status_ (&pi, dc);
2459 GNUNET_FS_free_download_request_ (dc->top_request);
2460 dc->top_request = NULL;
2461 if (dc->active != NULL)
2463 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2466 if (dc->filename != NULL)
2468 if ( (dc->completed != dc->length) &&
2469 (GNUNET_YES == do_delete) )
2471 if (0 != UNLINK (dc->filename))
2472 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
2476 GNUNET_free (dc->filename);
2478 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2479 GNUNET_FS_uri_destroy (dc->uri);
2480 if (NULL != dc->temp_filename)
2482 if (0 != UNLINK (dc->temp_filename))
2483 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
2486 GNUNET_free (dc->temp_filename);
2488 GNUNET_free_non_null (dc->serialization);
2492 /* end of fs_download.c */