2 This file is part of GNUnet.
3 (C) 2001-2011 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - different priority for scheduling probe downloads?
29 #include "gnunet_constants.h"
30 #include "gnunet_fs_service.h"
34 #define DEBUG_DOWNLOAD GNUNET_NO
37 * Determine if the given download (options and meta data) should cause
38 * use to try to do a recursive download.
41 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
43 return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
44 ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
45 ( (dc->meta == NULL) &&
46 ( (NULL == dc->filename) ||
47 ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
49 strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
50 GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
55 * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
56 * only have to truncate the file once we're done).
58 * Given the offset of a block (with respect to the DBLOCKS) and its
59 * depth, return the offset where we would store this block in the
62 * @param fsize overall file size
63 * @param off offset of the block in the file
64 * @param depth depth of the block in the tree, 0 for DBLOCK
65 * @return off for DBLOCKS (depth == treedepth),
66 * otherwise an offset past the end
67 * of the file that does not overlap
68 * with the range for any other block
71 compute_disk_offset (uint64_t fsize,
76 uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
77 uint64_t loff; /* where do IBlocks for depth "i" start? */
78 unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
82 /* first IBlocks start at the end of file, rounded up
83 to full DBLOCK_SIZE */
84 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
85 lsize = ( (fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
86 GNUNET_assert (0 == (off % DBLOCK_SIZE));
87 ioff = (off / DBLOCK_SIZE);
91 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
92 GNUNET_assert (lsize > 0);
93 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
94 ioff /= CHK_PER_INODE;
96 return loff + ioff * sizeof (struct ContentHashKey);
101 * Given a block at the given offset and depth, calculate the offset
102 * for the CHK at the given index.
104 * @param offset the offset of the first
105 * DBLOCK in the subtree of the
107 * @param depth the depth of the IBLOCK in the tree, 0 for DBLOCK
108 * @param k which CHK in the IBLOCK are we
110 * @return offset if k=0, otherwise an appropriately
111 * larger value (i.e., if depth = 1,
112 * the returned value should be offset+k*DBLOCK_SIZE)
115 compute_dblock_offset (uint64_t offset,
120 uint64_t lsize; /* what is the size of the sum of all DBlocks
121 that a CHK at depth i corresponds to? */
126 for (i=1;i<depth;i++)
127 lsize *= CHK_PER_INODE;
128 return offset + k * lsize;
133 * Fill in all of the generic fields for a download event and call the
136 * @param pi structure to fill in
137 * @param dc overall download context
140 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
141 struct GNUNET_FS_DownloadContext *dc)
143 pi->value.download.dc = dc;
144 pi->value.download.cctx
146 pi->value.download.pctx
147 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
148 pi->value.download.sctx
149 = (dc->search == NULL) ? NULL : dc->search->client_info;
150 pi->value.download.uri
152 pi->value.download.filename
154 pi->value.download.size
156 pi->value.download.duration
157 = GNUNET_TIME_absolute_get_duration (dc->start_time);
158 pi->value.download.completed
160 pi->value.download.anonymity
162 pi->value.download.eta
163 = GNUNET_TIME_calculate_eta (dc->start_time,
166 pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
167 if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
168 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
171 dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
177 * We're ready to transmit a search request to the
178 * file-sharing service. Do it. If there is
179 * more than one request pending, try to send
180 * multiple or request another transmission.
183 * @param size number of bytes available in buf
184 * @param buf where the callee should write the message
185 * @return number of bytes written to buf
188 transmit_download_request (void *cls,
194 * Closure for iterator processing results.
196 struct ProcessResultClosure
202 GNUNET_HashCode query;
205 * Data found in P2P network.
210 * Our download context.
212 struct GNUNET_FS_DownloadContext *dc;
215 * Number of bytes in data.
222 enum GNUNET_BLOCK_Type type;
225 * Flag to indicate if this block should be stored on disk.
233 * Iterator over entries in the pending requests in the 'active' map for the
234 * reply that we just got.
236 * @param cls closure (our 'struct ProcessResultClosure')
237 * @param key query for the given value / request
238 * @param value value in the hash map (a 'struct DownloadRequest')
239 * @return GNUNET_YES (we should continue to iterate); unless serious error
242 process_result_with_request (void *cls,
243 const GNUNET_HashCode * key,
248 * We've found a matching block without downloading it.
249 * Encrypt it and pass it to our "receive" function as
250 * if we had received it from the network.
252 * @param dc download in question
253 * @param chk request this relates to
254 * @param dr request details
255 * @param block plaintext data matching request
256 * @param len number of bytes in block
257 * @param do_store should we still store the block on disk?
258 * @return GNUNET_OK on success
261 encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc,
262 const struct ContentHashKey *chk,
263 struct DownloadRequest *dr,
268 struct ProcessResultClosure prc;
270 struct GNUNET_CRYPTO_AesSessionKey sk;
271 struct GNUNET_CRYPTO_AesInitializationVector iv;
272 GNUNET_HashCode query;
274 GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
275 if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
281 return GNUNET_SYSERR;
283 GNUNET_CRYPTO_hash (enc, len, &query);
284 if (0 != memcmp (&query,
286 sizeof (GNUNET_HashCode)))
289 return GNUNET_SYSERR;
292 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
293 "Matching block for `%s' at offset %llu already present, no need for download!\n",
295 (unsigned long long) dr->offset);
297 /* already got it! */
301 prc.type = (0 == dr->depth)
302 ? GNUNET_BLOCK_TYPE_FS_DBLOCK
303 : GNUNET_BLOCK_TYPE_FS_IBLOCK;
304 prc.query = chk->query;
305 prc.do_store = do_store;
306 process_result_with_request (&prc,
314 * We've lost our connection with the FS service.
315 * Re-establish it and re-transmit all of our
318 * @param dc download context that is having trouble
321 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
325 * We found an entry in a directory. Check if the respective child
326 * already exists and if not create the respective child download.
328 * @param cls the parent download
329 * @param filename name of the file in the directory
330 * @param uri URI of the file (CHK or LOC)
331 * @param meta meta data of the file
332 * @param length number of bytes in data
333 * @param data contents of the file (or NULL if they were not inlined)
336 trigger_recursive_download (void *cls,
337 const char *filename,
338 const struct GNUNET_FS_Uri *uri,
339 const struct GNUNET_CONTAINER_MetaData *meta,
345 * We're done downloading a directory. Open the file and
346 * trigger all of the (remaining) child downloads.
348 * @param dc context of download that just completed
351 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
356 struct GNUNET_DISK_FileHandle *h;
357 struct GNUNET_DISK_MapHandle *m;
359 size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
360 size = (size_t) size64;
361 if (size64 != (uint64_t) size)
363 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
364 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
367 if (dc->filename != NULL)
369 h = GNUNET_DISK_file_open (dc->filename,
370 GNUNET_DISK_OPEN_READ,
371 GNUNET_DISK_PERM_NONE);
375 GNUNET_assert (dc->temp_filename != NULL);
376 h = GNUNET_DISK_file_open (dc->temp_filename,
377 GNUNET_DISK_OPEN_READ,
378 GNUNET_DISK_PERM_NONE);
382 data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
385 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
386 _("Directory too large for system address space\n"));
390 GNUNET_FS_directory_list_contents (size,
393 &trigger_recursive_download,
395 GNUNET_DISK_file_unmap (m);
397 GNUNET_DISK_file_close (h);
398 if (dc->filename == NULL)
400 if (0 != UNLINK (dc->temp_filename))
401 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
404 GNUNET_free (dc->temp_filename);
405 dc->temp_filename = NULL;
411 * Check if all child-downloads have completed (or trigger them if
412 * necessary) and once we're completely done, signal completion (and
413 * possibly recurse to parent). This function MUST be called when the
414 * download of a file itself is done or when the download of a file is
415 * done and then later a direct child download has completed (and
416 * hence this download may complete itself).
418 * @param dc download to check for completion of children
421 check_completed (struct GNUNET_FS_DownloadContext *dc)
423 struct GNUNET_FS_ProgressInfo pi;
424 struct GNUNET_FS_DownloadContext *pos;
426 /* first, check if we need to download children */
427 if ( (dc->child_head == NULL) &&
428 (is_recursive_download (dc)) )
429 full_recursive_download (dc);
430 /* then, check if children are done already */
431 pos = dc->child_head;
434 if ( (pos->emsg == NULL) &&
435 (pos->completed < pos->length) )
436 return; /* not done yet */
437 if ( (pos->child_head != NULL) &&
438 (pos->has_finished != GNUNET_YES) )
439 return; /* not transitively done yet */
442 /* All of our children are done, so mark this download done */
443 dc->has_finished = GNUNET_YES;
444 GNUNET_FS_download_sync_ (dc);
446 /* signal completion */
447 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
448 GNUNET_FS_download_make_status_ (&pi, dc);
450 /* let parent know */
451 if (dc->parent != NULL)
452 check_completed (dc->parent);
457 * We got a block of plaintext data (from the meta data).
458 * Try it for upward reconstruction of the data. On success,
459 * the top-level block will move to state BRS_DOWNLOAD_UP.
461 * @param dr one of our request entries
462 * @param data plaintext data, starting from the beginning of the file
463 * @param data_len number of bytes in data
466 try_match_block (struct GNUNET_FS_DownloadContext *dc,
467 struct DownloadRequest *dr,
471 struct GNUNET_FS_ProgressInfo pi;
473 char enc[DBLOCK_SIZE];
474 struct ContentHashKey chks[CHK_PER_INODE];
475 struct ContentHashKey in_chk;
476 struct GNUNET_CRYPTO_AesSessionKey sk;
477 struct GNUNET_CRYPTO_AesInitializationVector iv;
479 struct DownloadRequest *drc;
480 struct GNUNET_DISK_FileHandle *fh;
487 odata_len = data_len;
488 if (BRS_DOWNLOAD_UP == dr->state)
492 complete = GNUNET_YES;
493 for (i=0;i<dr->num_children;i++)
495 drc = dr->children[i];
499 if (drc->state != BRS_RECONSTRUCT_META_UP)
500 complete = GNUNET_NO;
504 if (GNUNET_YES != complete)
506 data = (const char*) chks;
507 dlen = dr->num_children * sizeof (struct ContentHashKey);
511 if (dr->offset > data_len)
513 dlen = GNUNET_MIN (data_len - dr->offset,
516 GNUNET_CRYPTO_hash (&data[dr->offset],
519 GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
520 if (-1 == GNUNET_CRYPTO_aes_encrypt (&data[dr->offset], dlen,
528 GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
533 dr->state = BRS_RECONSTRUCT_META_UP;
536 if (0 != memcmp (&in_chk,
538 sizeof (struct ContentHashKey)))
540 /* other peer provided bogus meta data */
544 /* write block to disk */
545 fn = dc->filename != NULL
548 fh = GNUNET_DISK_file_open (fn,
549 GNUNET_DISK_OPEN_READWRITE |
550 GNUNET_DISK_OPEN_CREATE |
551 GNUNET_DISK_OPEN_TRUNCATE,
552 GNUNET_DISK_PERM_USER_READ |
553 GNUNET_DISK_PERM_USER_WRITE |
554 GNUNET_DISK_PERM_GROUP_READ |
555 GNUNET_DISK_PERM_OTHER_READ);
558 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
561 GNUNET_asprintf (&dc->emsg,
562 _("Failed to open file `%s' for writing"),
564 GNUNET_DISK_file_close (fh);
565 dr->state = BRS_ERROR;
566 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
567 pi.value.download.specifics.error.message = dc->emsg;
568 GNUNET_FS_download_make_status_ (&pi, dc);
572 GNUNET_DISK_file_write (fh,
576 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
579 GNUNET_asprintf (&dc->emsg,
580 _("Failed to open file `%s' for writing"),
582 GNUNET_DISK_file_close (fh);
583 dr->state = BRS_ERROR;
584 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
585 pi.value.download.specifics.error.message = dc->emsg;
586 GNUNET_FS_download_make_status_ (&pi, dc);
589 GNUNET_DISK_file_close (fh);
591 dr->state = BRS_DOWNLOAD_UP;
592 dc->completed = dc->length;
593 GNUNET_FS_download_sync_ (dc);
594 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
595 pi.value.download.specifics.progress.data = data;
596 pi.value.download.specifics.progress.offset = 0;
597 pi.value.download.specifics.progress.data_len = dlen;
598 pi.value.download.specifics.progress.depth = 0;
599 GNUNET_FS_download_make_status_ (&pi, dc);
600 if (0 != truncate (dc->filename,
601 GNUNET_ntohll (dc->uri->data.chk.file_length)))
602 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
605 check_completed (dc);
608 /* how did we get here? */
616 * Type of a function that libextractor calls for each
617 * meta data item found. If we find full data meta data,
618 * call 'try_match_block' on it.
620 * @param cls our 'struct GNUNET_FS_DownloadContext*'
621 * @param plugin_name name of the plugin that produced this value;
622 * special values can be used (i.e. '<zlib>' for zlib being
623 * used in the main libextractor library and yielding
625 * @param type libextractor-type describing the meta data
626 * @param format basic format information about data
627 * @param data_mime_type mime-type of data (not of the original file);
628 * can be NULL (if mime-type is not known)
629 * @param data actual meta-data found
630 * @param data_len number of bytes in data
631 * @return 0 to continue extracting, 1 to abort
634 match_full_data (void *cls,
635 const char *plugin_name,
636 enum EXTRACTOR_MetaType type,
637 enum EXTRACTOR_MetaFormat format,
638 const char *data_mime_type,
642 struct GNUNET_FS_DownloadContext *dc = cls;
644 if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
647 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
648 "Found %u bytes of FD!\n",
649 (unsigned int) data_len);
651 if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
654 return 1; /* bogus meta data */
665 * Set the state of the given download request to
666 * BRS_DOWNLOAD_UP and propagate it up the tree.
668 * @param dr download request that is done
671 propagate_up (struct DownloadRequest *dr)
677 dr->state = BRS_DOWNLOAD_UP;
681 for (i=0;i<dr->num_children;i++)
682 if (dr->children[i]->state != BRS_DOWNLOAD_UP)
685 while (i == dr->num_children);
690 * Try top-down reconstruction. Before, the given request node
691 * must have the state BRS_CHK_SET. Afterwards, more nodes may
692 * have that state or advanced to BRS_DOWNLOAD_DOWN or even
693 * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the
696 * @param dc overall download this block belongs to
697 * @param dr block to reconstruct
700 try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
701 struct DownloadRequest *dr)
704 char block[DBLOCK_SIZE];
709 unsigned int chk_off;
710 struct DownloadRequest *drc;
711 uint64_t child_block_size;
712 const struct ContentHashKey *chks;
715 GNUNET_assert (dc->rfh != NULL);
716 GNUNET_assert (dr->state == BRS_CHK_SET);
717 total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
718 GNUNET_assert (dr->depth < dc->treedepth);
719 len = GNUNET_FS_tree_calculate_block_size (total,
722 GNUNET_assert (len <= DBLOCK_SIZE);
723 off = compute_disk_offset (total,
726 if (dc->old_file_size < off + len)
727 return; /* failure */
729 GNUNET_DISK_file_seek (dc->rfh,
731 GNUNET_DISK_SEEK_SET) )
733 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
736 return; /* failure */
739 GNUNET_DISK_file_read (dc->rfh,
743 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
746 return; /* failure */
748 GNUNET_CRYPTO_hash (block, len, &key);
749 if (0 != memcmp (&key,
751 sizeof (GNUNET_HashCode)))
752 return; /* mismatch */
754 encrypt_existing_match (dc,
761 /* hash matches but encrypted block does not, really bad */
762 dr->state = BRS_ERROR;
764 while (dr->parent != NULL)
767 dr->state = BRS_ERROR;
772 dr->state = BRS_DOWNLOAD_DOWN;
774 /* set CHKs for children */
775 up_done = GNUNET_YES;
776 chks = (const struct ContentHashKey*) block;
777 for (i=0;i<dr->num_children;i++)
779 drc = dr->children[i];
780 GNUNET_assert (drc->offset >= dr->offset);
781 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
782 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
783 chk_off = (drc->offset - dr->offset) / child_block_size;
784 GNUNET_assert (drc->state == BRS_INIT);
785 drc->state = BRS_CHK_SET;
786 drc->chk = chks[chk_off];
787 try_top_down_reconstruction (dc, drc);
788 if (drc->state != BRS_DOWNLOAD_UP)
789 up_done = GNUNET_NO; /* children not all done */
791 if (up_done == GNUNET_YES)
792 propagate_up (dr); /* children all done (or no children...) */
797 * Schedule the download of the specified block in the tree.
799 * @param dc overall download this block belongs to
800 * @param chk content-hash-key of the block
801 * @param offset offset of the block in the file
802 * (for IBlocks, the offset is the lowest
803 * offset of any DBlock in the subtree under
805 * @param depth depth of the block, 0 is the root of the tree
808 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
809 struct DownloadRequest *dr)
818 case BRS_RECONSTRUCT_DOWN:
821 case BRS_RECONSTRUCT_META_UP:
824 case BRS_RECONSTRUCT_UP:
828 /* normal case, start download */
830 case BRS_DOWNLOAD_DOWN:
831 for (i=0;i<dr->num_children;i++)
832 schedule_block_download (dc, dr->children[i]);
834 case BRS_DOWNLOAD_UP:
842 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
843 "Scheduling download at offset %llu and depth %u for `%s'\n",
844 (unsigned long long) dr->offset,
846 GNUNET_h2s (&dr->chk.query));
848 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
851 dr->is_pending = GNUNET_YES;
852 GNUNET_CONTAINER_multihashmap_put (dc->active,
855 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
856 if (dc->client == NULL)
857 return; /* download not active */
859 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
860 sizeof (struct SearchMessage),
861 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
863 &transmit_download_request,
868 #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
871 * We found an entry in a directory. Check if the respective child
872 * already exists and if not create the respective child download.
874 * @param cls the parent download
875 * @param filename name of the file in the directory
876 * @param uri URI of the file (CHK or LOC)
877 * @param meta meta data of the file
878 * @param length number of bytes in data
879 * @param data contents of the file (or NULL if they were not inlined)
882 trigger_recursive_download (void *cls,
883 const char *filename,
884 const struct GNUNET_FS_Uri *uri,
885 const struct GNUNET_CONTAINER_MetaData *meta,
889 struct GNUNET_FS_DownloadContext *dc = cls;
890 struct GNUNET_FS_DownloadContext *cpos;
891 struct GNUNET_DISK_FileHandle *fh;
893 const char *real_name;
902 return; /* entry for the directory itself */
903 cpos = dc->child_head;
906 if ( (GNUNET_FS_uri_test_equal (uri,
908 ( (filename != NULL) &&
909 (0 == strcmp (cpos->filename,
915 return; /* already exists */
917 if (NULL == filename)
919 fn = GNUNET_FS_meta_data_suggest_filename (meta);
922 us = GNUNET_FS_uri_to_string (uri);
923 fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_CHK_PREFIX)]);
926 else if (fn[0] == '.')
929 us = GNUNET_FS_uri_to_string (uri);
930 GNUNET_asprintf (&fn,
932 &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], ext);
936 /* change '\' to '/' (this should have happened
937 during insertion, but malicious peers may
938 not have done this) */
939 while (NULL != (pos = strstr (fn, "\\")))
941 /* remove '../' everywhere (again, well-behaved
942 peers don't do this, but don't trust that
943 we did not get something nasty) */
944 while (NULL != (pos = strstr (fn, "../")))
952 if (dc->filename == NULL)
958 dn = GNUNET_strdup (dc->filename);
959 GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
961 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
962 GNUNET_FS_DIRECTORY_EXT)) );
963 if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
965 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
966 GNUNET_FS_DIRECTORY_EXT)) )
967 dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
968 if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
969 ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
971 strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
972 GNUNET_FS_DIRECTORY_EXT)) ) )
974 GNUNET_asprintf (&full_name,
979 GNUNET_FS_DIRECTORY_EXT);
983 GNUNET_asprintf (&full_name,
991 if ( (full_name != NULL) &&
993 GNUNET_DISK_directory_create_for_file (full_name)) )
995 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
996 _("Failed to create directory for recursive download of `%s'\n"),
998 GNUNET_free (full_name);
999 GNUNET_free_non_null (fn);
1004 if ( (data != NULL) &&
1005 (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
1007 if (full_name == NULL)
1009 temp_name = GNUNET_DISK_mktemp ("gnunet-download-trd");
1010 real_name = temp_name;
1014 real_name = full_name;
1016 /* write to disk, then trigger normal download which will instantly progress to completion */
1017 fh = GNUNET_DISK_file_open (real_name,
1018 GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
1019 GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
1022 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1025 GNUNET_free (full_name);
1026 GNUNET_free_non_null (fn);
1030 GNUNET_DISK_file_write (fh,
1034 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1038 GNUNET_DISK_file_close (fh);
1041 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1042 "Triggering recursive download of size %llu with %u bytes MD\n",
1043 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
1044 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (meta));
1046 GNUNET_FS_download_start (dc->h,
1049 full_name, temp_name,
1051 GNUNET_FS_uri_chk_get_file_size (uri),
1056 GNUNET_free_non_null (full_name);
1057 GNUNET_free_non_null (temp_name);
1058 GNUNET_free_non_null (fn);
1063 * (recursively) free download request structure
1065 * @param dr request to free
1068 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
1074 for (i=0;i<dr->num_children;i++)
1075 GNUNET_FS_free_download_request_ (dr->children[i]);
1076 GNUNET_free_non_null (dr->children);
1082 * Iterator over entries in the pending requests in the 'active' map for the
1083 * reply that we just got.
1085 * @param cls closure (our 'struct ProcessResultClosure')
1086 * @param key query for the given value / request
1087 * @param value value in the hash map (a 'struct DownloadRequest')
1088 * @return GNUNET_YES (we should continue to iterate); unless serious error
1091 process_result_with_request (void *cls,
1092 const GNUNET_HashCode *key,
1095 struct ProcessResultClosure *prc = cls;
1096 struct DownloadRequest *dr = value;
1097 struct GNUNET_FS_DownloadContext *dc = prc->dc;
1098 struct DownloadRequest *drc;
1099 struct GNUNET_DISK_FileHandle *fh = NULL;
1100 struct GNUNET_CRYPTO_AesSessionKey skey;
1101 struct GNUNET_CRYPTO_AesInitializationVector iv;
1103 struct GNUNET_FS_ProgressInfo pi;
1108 struct ContentHashKey *chk;
1111 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1112 "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n",
1115 (unsigned long long) dr->offset,
1116 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1119 bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
1122 if (prc->size != bs)
1124 GNUNET_asprintf (&dc->emsg,
1125 _("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"),
1128 (unsigned long long) dr->offset,
1129 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
1131 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1134 while (dr->parent != NULL)
1136 dr->state = BRS_ERROR;
1139 dr->state = BRS_ERROR;
1143 (void) GNUNET_CONTAINER_multihashmap_remove (dc->active,
1146 if (GNUNET_YES == dr->is_pending)
1148 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1151 dr->is_pending = GNUNET_NO;
1155 GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
1156 if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
1163 dc->emsg = GNUNET_strdup (_("internal error decrypting content"));
1166 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
1170 if ( ( GNUNET_YES == prc->do_store) &&
1171 ( (dc->filename != NULL) ||
1172 (is_recursive_download (dc)) ) &&
1173 ( (dr->depth == dc->treedepth) ||
1174 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
1176 fh = GNUNET_DISK_file_open (dc->filename != NULL
1178 : dc->temp_filename,
1179 GNUNET_DISK_OPEN_READWRITE |
1180 GNUNET_DISK_OPEN_CREATE,
1181 GNUNET_DISK_PERM_USER_READ |
1182 GNUNET_DISK_PERM_USER_WRITE |
1183 GNUNET_DISK_PERM_GROUP_READ |
1184 GNUNET_DISK_PERM_OTHER_READ);
1187 GNUNET_asprintf (&dc->emsg,
1188 _("Download failed: could not open file `%s': %s\n"),
1194 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1195 "Saving decrypted block to disk at offset %llu\n",
1196 (unsigned long long) off);
1199 GNUNET_DISK_file_seek (fh,
1201 GNUNET_DISK_SEEK_SET) ) )
1203 GNUNET_asprintf (&dc->emsg,
1204 _("Failed to seek to offset %llu in file `%s': %s\n"),
1205 (unsigned long long) off,
1211 GNUNET_DISK_file_write (fh,
1215 GNUNET_asprintf (&dc->emsg,
1216 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1217 (unsigned int) prc->size,
1218 (unsigned long long) off,
1223 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1229 /* DBLOCK, update progress and try recursion if applicable */
1231 if (dr->offset < dc->offset)
1233 /* starting offset begins in the middle of pt,
1234 do not count first bytes as progress */
1235 GNUNET_assert (app > (dc->offset - dr->offset));
1236 app -= (dc->offset - dr->offset);
1238 if (dr->offset + prc->size > dc->offset + dc->length)
1240 /* end of block is after relevant range,
1241 do not count last bytes as progress */
1242 GNUNET_assert (app > (dr->offset + prc->size) - (dc->offset + dc->length));
1243 app -= (dr->offset + prc->size) - (dc->offset + dc->length);
1245 dc->completed += app;
1247 /* do recursive download if option is set and either meta data
1248 says it is a directory or if no meta data is given AND filename
1249 ends in '.gnd' (top-level case) */
1250 if (is_recursive_download (dc))
1251 GNUNET_FS_directory_list_contents (prc->size,
1254 &trigger_recursive_download,
1258 dr->state = BRS_DOWNLOAD_DOWN;
1259 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1260 pi.value.download.specifics.progress.data = pt;
1261 pi.value.download.specifics.progress.offset = dr->offset;
1262 pi.value.download.specifics.progress.data_len = prc->size;
1263 pi.value.download.specifics.progress.depth = dr->depth;
1264 GNUNET_FS_download_make_status_ (&pi, dc);
1265 GNUNET_assert (dc->completed <= dc->length);
1267 if (dc->completed == dc->length)
1269 /* download completed, signal */
1271 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1272 "Download completed, truncating file to desired length %llu\n",
1273 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1275 /* truncate file to size (since we store IBlocks at the end) */
1276 if (dc->filename != NULL)
1278 if (0 != truncate (dc->filename,
1279 GNUNET_ntohll (dc->uri->data.chk.file_length)))
1280 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1284 if (dc->job_queue != NULL)
1286 GNUNET_FS_dequeue_ (dc->job_queue);
1287 dc->job_queue = NULL;
1289 GNUNET_assert (dr->depth == 0);
1290 check_completed (dc);
1295 /* bottom of the tree, no child downloads possible, just sync */
1296 GNUNET_FS_download_sync_ (dc);
1301 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1302 "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1304 (unsigned long long) dr->offset);
1306 GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1307 chk = (struct ContentHashKey*) pt;
1308 for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1310 off = compute_dblock_offset (dr->offset,
1313 drc = dr->children[i];
1318 drc->state = BRS_CHK_SET;
1319 schedule_block_download (dc, drc);
1321 case BRS_RECONSTRUCT_DOWN:
1324 case BRS_RECONSTRUCT_META_UP:
1327 case BRS_RECONSTRUCT_UP:
1333 case BRS_DOWNLOAD_DOWN:
1336 case BRS_DOWNLOAD_UP:
1347 GNUNET_FS_download_sync_ (dc);
1352 GNUNET_DISK_file_close (fh);
1353 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1354 pi.value.download.specifics.error.message = dc->emsg;
1355 GNUNET_FS_download_make_status_ (&pi, dc);
1356 /* abort all pending requests */
1359 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1362 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1364 GNUNET_FS_free_download_request_ (dc->top_request);
1365 dc->top_request = NULL;
1366 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1368 dc->pending_head = NULL;
1369 dc->pending_tail = NULL;
1370 GNUNET_FS_download_sync_ (dc);
1376 * Process a download result.
1378 * @param dc our download context
1379 * @param type type of the result
1380 * @param data the (encrypted) response
1381 * @param size size of data
1384 process_result (struct GNUNET_FS_DownloadContext *dc,
1385 enum GNUNET_BLOCK_Type type,
1389 struct ProcessResultClosure prc;
1395 prc.do_store = GNUNET_YES;
1396 GNUNET_CRYPTO_hash (data, size, &prc.query);
1398 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1399 "Received result for query `%s' from `%s'-service\n",
1400 GNUNET_h2s (&prc.query),
1403 GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1405 &process_result_with_request,
1411 * Type of a function to call when we receive a message
1414 * @param cls closure
1415 * @param msg message received, NULL on timeout or fatal error
1418 receive_results (void *cls,
1419 const struct GNUNET_MessageHeader * msg)
1421 struct GNUNET_FS_DownloadContext *dc = cls;
1422 const struct PutMessage *cm;
1425 if ( (NULL == msg) ||
1426 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1427 (sizeof (struct PutMessage) > ntohs(msg->size)) )
1429 GNUNET_break (msg == NULL);
1433 msize = ntohs(msg->size);
1434 cm = (const struct PutMessage*) msg;
1438 msize - sizeof (struct PutMessage));
1439 if (dc->client == NULL)
1440 return; /* fatal error */
1441 /* continue receiving */
1442 GNUNET_CLIENT_receive (dc->client,
1445 GNUNET_TIME_UNIT_FOREVER_REL);
1451 * We're ready to transmit a search request to the
1452 * file-sharing service. Do it. If there is
1453 * more than one request pending, try to send
1454 * multiple or request another transmission.
1456 * @param cls closure
1457 * @param size number of bytes available in buf
1458 * @param buf where the callee should write the message
1459 * @return number of bytes written to buf
1462 transmit_download_request (void *cls,
1466 struct GNUNET_FS_DownloadContext *dc = cls;
1468 struct SearchMessage *sm;
1469 struct DownloadRequest *dr;
1475 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1476 "Transmitting download request failed, trying to reconnect\n");
1481 GNUNET_assert (size >= sizeof (struct SearchMessage));
1484 while ( (NULL != (dr = dc->pending_head)) &&
1485 (size >= msize + sizeof (struct SearchMessage)) )
1488 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1489 "Transmitting download request for `%s' to `%s'-service\n",
1490 GNUNET_h2s (&dr->chk.query),
1493 memset (sm, 0, sizeof (struct SearchMessage));
1494 sm->header.size = htons (sizeof (struct SearchMessage));
1495 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1496 if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1497 sm->options = htonl (1);
1499 sm->options = htonl (0);
1501 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
1503 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
1504 sm->anonymity_level = htonl (dc->anonymity);
1505 sm->target = dc->target.hashPubKey;
1506 sm->query = dr->chk.query;
1507 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1510 dr->is_pending = GNUNET_NO;
1511 msize += sizeof (struct SearchMessage);
1514 if (dc->pending_head != NULL)
1516 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1517 sizeof (struct SearchMessage),
1518 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1520 &transmit_download_request,
1522 GNUNET_assert (dc->th != NULL);
1529 * Reconnect to the FS service and transmit our queries NOW.
1531 * @param cls our download context
1535 do_reconnect (void *cls,
1536 const struct GNUNET_SCHEDULER_TaskContext *tc)
1538 struct GNUNET_FS_DownloadContext *dc = cls;
1539 struct GNUNET_CLIENT_Connection *client;
1541 dc->task = GNUNET_SCHEDULER_NO_TASK;
1542 client = GNUNET_CLIENT_connect ("fs",
1546 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1547 "Connecting to `%s'-service failed, will try again.\n",
1552 dc->client = client;
1553 if (dc->pending_head != NULL)
1555 dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1556 sizeof (struct SearchMessage),
1557 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1559 &transmit_download_request,
1561 GNUNET_assert (dc->th != NULL);
1563 GNUNET_CLIENT_receive (client,
1566 GNUNET_TIME_UNIT_FOREVER_REL);
1571 * Add entries to the pending list.
1573 * @param cls our download context
1575 * @param entry entry of type "struct DownloadRequest"
1579 retry_entry (void *cls,
1580 const GNUNET_HashCode *key,
1583 struct GNUNET_FS_DownloadContext *dc = cls;
1584 struct DownloadRequest *dr = entry;
1586 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
1589 dr->is_pending = GNUNET_YES;
1595 * We've lost our connection with the FS service.
1596 * Re-establish it and re-transmit all of our
1599 * @param dc download context that is having trouble
1602 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1605 if (NULL != dc->client)
1608 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1609 "Moving all requests back to pending list\n");
1613 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1616 /* full reset of the pending list */
1617 dc->pending_head = NULL;
1618 dc->pending_tail = NULL;
1619 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1622 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1626 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1627 "Will try to reconnect in 1s\n");
1630 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1637 * We're allowed to ask the FS service for our blocks. Start the download.
1639 * @param cls the 'struct GNUNET_FS_DownloadContext'
1640 * @param client handle to use for communcation with FS (we must destroy it!)
1643 activate_fs_download (void *cls,
1644 struct GNUNET_CLIENT_Connection *client)
1646 struct GNUNET_FS_DownloadContext *dc = cls;
1647 struct GNUNET_FS_ProgressInfo pi;
1650 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1651 "Download activated\n");
1653 GNUNET_assert (NULL != client);
1654 GNUNET_assert (dc->client == NULL);
1655 GNUNET_assert (dc->th == NULL);
1656 dc->client = client;
1657 GNUNET_CLIENT_receive (client,
1660 GNUNET_TIME_UNIT_FOREVER_REL);
1661 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1662 GNUNET_FS_download_make_status_ (&pi, dc);
1663 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1667 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1668 "Asking for transmission to FS service\n");
1670 if (dc->pending_head != NULL)
1672 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1673 sizeof (struct SearchMessage),
1674 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1676 &transmit_download_request,
1678 GNUNET_assert (dc->th != NULL);
1684 * We must stop to ask the FS service for our blocks. Pause the download.
1686 * @param cls the 'struct GNUNET_FS_DownloadContext'
1689 deactivate_fs_download (void *cls)
1691 struct GNUNET_FS_DownloadContext *dc = cls;
1692 struct GNUNET_FS_ProgressInfo pi;
1695 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1696 "Download deactivated\n");
1700 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1703 if (NULL != dc->client)
1705 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1708 pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1709 GNUNET_FS_download_make_status_ (&pi, dc);
1714 * (recursively) Create a download request structure.
1716 * @param parent parent of the current entry
1717 * @param depth depth of the current entry, 0 are the DBLOCKs,
1718 * top level block is 'dc->treedepth - 1'
1719 * @param dr_offset offset in the original file this block maps to
1720 * (as in, offset of the first byte of the first DBLOCK
1721 * in the subtree rooted in the returned download request tree)
1722 * @param file_start_offset desired starting offset for the download
1723 * in the original file; requesting tree should not contain
1724 * DBLOCKs prior to the file_start_offset
1725 * @param file_length desired number of bytes the user wanted to access
1726 * (from file_start_offset). Resulting tree should not contain
1727 * DBLOCKs after file_start_offset + file_length.
1728 * @return download request tree for the given range of DBLOCKs at
1729 * the specified depth
1731 static struct DownloadRequest *
1732 create_download_request (struct DownloadRequest *parent,
1735 uint64_t file_start_offset,
1736 uint64_t desired_length)
1738 struct DownloadRequest *dr;
1740 unsigned int head_skip;
1741 uint64_t child_block_size;
1743 dr = GNUNET_malloc (sizeof (struct DownloadRequest));
1744 dr->parent = parent;
1746 dr->offset = dr_offset;
1749 child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
1751 /* calculate how many blocks at this level are not interesting
1752 from the start (rounded down), either because of the requested
1753 file offset or because this IBlock is further along */
1754 if (dr_offset < file_start_offset)
1755 head_skip = file_start_offset / child_block_size;
1757 head_skip = dr_offset / child_block_size;
1759 /* calculate index of last block at this level that is interesting (rounded up) */
1760 dr->num_children = file_start_offset + desired_length / child_block_size;
1761 if (dr->num_children * child_block_size < file_start_offset + desired_length)
1762 dr->num_children++; /* round up */
1764 /* now we can get the total number of children for this block */
1765 dr->num_children -= head_skip;
1766 if (dr->num_children > CHK_PER_INODE)
1767 dr->num_children = CHK_PER_INODE; /* cap at max */
1769 /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
1770 GNUNET_assert (dr->num_children > 0);
1772 dr->children = GNUNET_malloc (dr->num_children *
1773 sizeof (struct DownloadRequest *));
1774 for (i=0;i<dr->num_children;i++)
1775 dr->children[i] = create_download_request (dr,
1777 dr_offset + i * child_block_size,
1786 * Continuation after a possible attempt to reconstruct
1787 * the current IBlock from the existing file.
1789 * @param cls the 'struct ReconstructContext'
1790 * @param tc scheduler context
1793 reconstruct_cont (void *cls,
1794 const struct GNUNET_SCHEDULER_TaskContext *tc)
1796 struct GNUNET_FS_DownloadContext *dc = cls;
1798 /* clean up state from tree encoder */
1801 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
1804 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
1806 GNUNET_SCHEDULER_cancel (dc->task);
1807 dc->task = GNUNET_SCHEDULER_NO_TASK;
1809 if (dc->rfh != NULL)
1811 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
1814 /* start "normal" download */
1815 schedule_block_download (dc,
1821 * Task requesting the next block from the tree encoder.
1823 * @param tc task context
1826 get_next_block (void *cls,
1827 const struct GNUNET_SCHEDULER_TaskContext *tc)
1829 struct GNUNET_FS_DownloadContext *dc = cls;
1831 dc->task = GNUNET_SCHEDULER_NO_TASK;
1832 GNUNET_FS_tree_encoder_next (dc->te);
1838 * Function called asking for the current (encoded)
1839 * block to be processed. After processing the
1840 * client should either call "GNUNET_FS_tree_encode_next"
1841 * or (on error) "GNUNET_FS_tree_encode_finish".
1843 * This function checks if the content on disk matches
1844 * the expected content based on the URI.
1846 * @param cls closure
1847 * @param chk content hash key for the block
1848 * @param offset offset of the block
1849 * @param depth depth of the block, 0 for DBLOCK
1850 * @param type type of the block (IBLOCK or DBLOCK)
1851 * @param block the (encrypted) block
1852 * @param block_size size of block (in bytes)
1855 reconstruct_cb (void *cls,
1856 const struct ContentHashKey *chk,
1859 enum GNUNET_BLOCK_Type type,
1861 uint16_t block_size)
1863 struct GNUNET_FS_DownloadContext *dc = cls;
1864 struct GNUNET_FS_ProgressInfo pi;
1865 struct DownloadRequest *dr;
1869 /* find corresponding request entry */
1870 dr = dc->top_request;
1871 while (dr->depth > depth)
1873 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1874 chld = (offset - dr->offset) / blen;
1875 GNUNET_assert (chld < dr->num_children);
1876 dr = dr->children[chld];
1882 case BRS_RECONSTRUCT_DOWN:
1884 case BRS_RECONSTRUCT_META_UP:
1886 case BRS_RECONSTRUCT_UP:
1889 if (0 == memcmp (chk,
1891 sizeof (struct ContentHashKey)))
1893 /* block matches, hence tree below matches;
1894 this request is done! */
1895 dr->state = BRS_DOWNLOAD_UP;
1896 /* calculate how many bytes of payload this block
1898 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1899 /* how many of those bytes are in the requested range? */
1900 blen = GNUNET_MIN (blen,
1901 dc->length + dc->offset - dr->offset);
1902 /* signal progress */
1903 dc->completed += blen;
1904 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1905 pi.value.download.specifics.progress.data = NULL;
1906 pi.value.download.specifics.progress.offset = offset;
1907 pi.value.download.specifics.progress.data_len = 0;
1908 pi.value.download.specifics.progress.depth = 0;
1909 GNUNET_FS_download_make_status_ (&pi, dc);
1915 case BRS_DOWNLOAD_DOWN:
1917 case BRS_DOWNLOAD_UP:
1925 if ( (dr == dc->top_request) &&
1926 (dr->state == BRS_DOWNLOAD_UP) )
1928 check_completed (dc);
1931 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
1937 * Function called by the tree encoder to obtain a block of plaintext
1938 * data (for the lowest level of the tree).
1940 * @param cls our 'struct ReconstructContext'
1941 * @param offset identifies which block to get
1942 * @param max (maximum) number of bytes to get; returning
1943 * fewer will also cause errors
1944 * @param buf where to copy the plaintext buffer
1945 * @param emsg location to store an error message (on error)
1946 * @return number of bytes copied to buf, 0 on error
1949 fh_reader (void *cls,
1955 struct GNUNET_FS_DownloadContext *dc = cls;
1956 struct GNUNET_DISK_FileHandle *fh = dc->rfh;
1961 GNUNET_DISK_file_seek (fh,
1963 GNUNET_DISK_SEEK_SET))
1965 *emsg = GNUNET_strdup (strerror (errno));
1968 ret = GNUNET_DISK_file_read (fh, buf, max);
1971 *emsg = GNUNET_strdup (strerror (errno));
1979 * Task that creates the initial (top-level) download
1980 * request for the file.
1982 * @param cls the 'struct GNUNET_FS_DownloadContext'
1983 * @param tc scheduler context
1986 GNUNET_FS_download_start_task_ (void *cls,
1987 const struct GNUNET_SCHEDULER_TaskContext *tc)
1989 struct GNUNET_FS_DownloadContext *dc = cls;
1990 struct GNUNET_FS_ProgressInfo pi;
1991 struct GNUNET_DISK_FileHandle *fh;
1994 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1995 "Start task running...\n");
1997 dc->task = GNUNET_SCHEDULER_NO_TASK;
1998 if (dc->length == 0)
2000 /* no bytes required! */
2001 if (dc->filename != NULL)
2003 fh = GNUNET_DISK_file_open (dc->filename != NULL
2005 : dc->temp_filename,
2006 GNUNET_DISK_OPEN_READWRITE |
2007 GNUNET_DISK_OPEN_CREATE |
2008 ( (0 == GNUNET_FS_uri_chk_get_file_size (dc->uri))
2009 ? GNUNET_DISK_OPEN_TRUNCATE : 0),
2010 GNUNET_DISK_PERM_USER_READ |
2011 GNUNET_DISK_PERM_USER_WRITE |
2012 GNUNET_DISK_PERM_GROUP_READ |
2013 GNUNET_DISK_PERM_OTHER_READ);
2014 GNUNET_DISK_file_close (fh);
2016 GNUNET_FS_download_sync_ (dc);
2017 check_completed (dc);
2020 if (dc->emsg != NULL)
2022 if (dc->top_request == NULL)
2024 dc->top_request = create_download_request (NULL, dc->treedepth - 1, 0,
2025 dc->offset, dc->length);
2026 dc->top_request->state = BRS_CHK_SET;
2027 dc->top_request->chk = (dc->uri->type == chk)
2028 ? dc->uri->data.chk.chk
2029 : dc->uri->data.loc.fi.chk;
2031 GNUNET_FS_download_sync_ (dc);
2032 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
2033 pi.value.download.specifics.start.meta = dc->meta;
2034 GNUNET_FS_download_make_status_ (&pi, dc);
2036 GNUNET_FS_download_start_downloading_ (dc);
2037 /* attempt reconstruction from disk */
2038 if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
2039 dc->rfh = GNUNET_DISK_file_open (dc->filename,
2040 GNUNET_DISK_OPEN_READ,
2041 GNUNET_DISK_PERM_NONE);
2042 if (dc->top_request->state == BRS_CHK_SET)
2044 if (dc->rfh != NULL)
2046 /* first, try top-down */
2047 try_top_down_reconstruction (dc, dc->top_request);
2048 switch (dc->top_request->state)
2052 case BRS_DOWNLOAD_DOWN:
2053 break; /* normal, some blocks already down */
2054 case BRS_DOWNLOAD_UP:
2055 /* already done entirely, party! */
2056 dc->completed = dc->length;
2057 GNUNET_FS_download_sync_ (dc);
2058 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
2059 /* slightly ugly: no data provided to callee; maybe mmap the
2060 file instead? Or is 'data' pure convenience!? */
2061 pi.value.download.specifics.progress.data = NULL;
2062 pi.value.download.specifics.progress.offset = dc->offset;
2063 pi.value.download.specifics.progress.data_len = dc->length;
2064 pi.value.download.specifics.progress.depth = 0;
2065 GNUNET_FS_download_make_status_ (&pi, dc);
2066 if (dc->rfh != NULL)
2068 /* avoid hanging on to file handle longer than
2070 GNUNET_DISK_file_close (dc->rfh);
2073 check_completed (dc);
2076 GNUNET_asprintf (&dc->emsg,
2078 GNUNET_FS_download_sync_ (dc);
2079 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
2080 pi.value.download.specifics.error.message = dc->emsg;
2081 GNUNET_FS_download_make_status_ (&pi, dc);
2089 /* attempt reconstruction from meta data */
2090 if ( (GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) &&
2091 (NULL != dc->meta) )
2094 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2095 "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
2096 (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
2097 (unsigned int) GNUNET_CONTAINER_meta_data_get_serialized_size (dc->meta));
2099 GNUNET_CONTAINER_meta_data_iterate (dc->meta,
2102 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2104 if (dc->rfh != NULL)
2106 /* avoid hanging on to file handle longer than
2108 GNUNET_DISK_file_close (dc->rfh);
2111 return; /* finished, status update was already done for us */
2114 if (dc->rfh != NULL)
2116 /* finally, try bottom-up */
2117 dc->te = GNUNET_FS_tree_encoder_create (dc->h,
2124 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
2129 /* simple, top-level download */
2130 schedule_block_download (dc,
2133 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2134 check_completed (dc);
2139 * Create SUSPEND event for the given download operation
2140 * and then clean up our state (without stop signal).
2142 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
2145 GNUNET_FS_download_signal_suspend_ (void *cls)
2147 struct GNUNET_FS_DownloadContext *dc = cls;
2148 struct GNUNET_FS_ProgressInfo pi;
2150 if (dc->top != NULL)
2151 GNUNET_FS_end_top (dc->h, dc->top);
2152 while (NULL != dc->child_head)
2153 GNUNET_FS_download_signal_suspend_ (dc->child_head);
2154 if (dc->search != NULL)
2156 dc->search->download = NULL;
2159 if (dc->job_queue != NULL)
2161 GNUNET_FS_dequeue_ (dc->job_queue);
2162 dc->job_queue = NULL;
2164 if (dc->parent != NULL)
2165 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2166 dc->parent->child_tail,
2168 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2170 GNUNET_SCHEDULER_cancel (dc->task);
2171 dc->task = GNUNET_SCHEDULER_NO_TASK;
2173 pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
2174 GNUNET_FS_download_make_status_ (&pi, dc);
2177 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2180 if (dc->rfh != NULL)
2182 GNUNET_DISK_file_close (dc->rfh);
2185 GNUNET_FS_free_download_request_ (dc->top_request);
2186 if (dc->active != NULL)
2188 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2191 GNUNET_free_non_null (dc->filename);
2192 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2193 GNUNET_FS_uri_destroy (dc->uri);
2194 GNUNET_free_non_null (dc->temp_filename);
2195 GNUNET_free_non_null (dc->serialization);
2201 * Download parts of a file. Note that this will store
2202 * the blocks at the respective offset in the given file. Also, the
2203 * download is still using the blocking of the underlying FS
2204 * encoding. As a result, the download may *write* outside of the
2205 * given boundaries (if offset and length do not match the 32k FS
2206 * block boundaries). <p>
2208 * This function should be used to focus a download towards a
2209 * particular portion of the file (optimization), not to strictly
2210 * limit the download to exactly those bytes.
2212 * @param h handle to the file sharing subsystem
2213 * @param uri the URI of the file (determines what to download); CHK or LOC URI
2214 * @param meta known metadata for the file (can be NULL)
2215 * @param filename where to store the file, maybe NULL (then no file is
2216 * created on disk and data must be grabbed from the callbacks)
2217 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2218 * can be NULL (in which case we will pick a name if needed); the temporary file
2219 * may already exist, in which case we will try to use the data that is there and
2220 * if it is not what is desired, will overwrite it
2221 * @param offset at what offset should we start the download (typically 0)
2222 * @param length how many bytes should be downloaded starting at offset
2223 * @param anonymity anonymity level to use for the download
2224 * @param options various options
2225 * @param cctx initial value for the client context for this download
2226 * @param parent parent download to associate this download with (use NULL
2227 * for top-level downloads; useful for manually-triggered recursive downloads)
2228 * @return context that can be used to control this download
2230 struct GNUNET_FS_DownloadContext *
2231 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
2232 const struct GNUNET_FS_Uri *uri,
2233 const struct GNUNET_CONTAINER_MetaData *meta,
2234 const char *filename,
2235 const char *tempname,
2239 enum GNUNET_FS_DownloadOptions options,
2241 struct GNUNET_FS_DownloadContext *parent)
2243 struct GNUNET_FS_DownloadContext *dc;
2245 GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
2246 GNUNET_FS_uri_test_loc (uri) );
2248 if ( (offset + length < offset) ||
2249 (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
2255 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2256 "Starting download `%s' of %llu bytes\n",
2258 (unsigned long long) length);
2260 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2262 dc->parent = parent;
2265 GNUNET_CONTAINER_DLL_insert (parent->child_head,
2269 dc->uri = GNUNET_FS_uri_dup (uri);
2270 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
2271 dc->client_info = cctx;
2272 dc->start_time = GNUNET_TIME_absolute_get ();
2273 if (NULL != filename)
2275 dc->filename = GNUNET_strdup (filename);
2276 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2277 GNUNET_DISK_file_size (filename,
2281 if (GNUNET_FS_uri_test_loc (dc->uri))
2282 GNUNET_assert (GNUNET_OK ==
2283 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2285 dc->offset = offset;
2286 dc->length = length;
2287 dc->anonymity = anonymity;
2288 dc->options = options;
2289 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2290 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
2291 if ( (filename == NULL) &&
2292 (is_recursive_download (dc) ) )
2294 if (tempname != NULL)
2295 dc->temp_filename = GNUNET_strdup (tempname);
2297 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2301 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2302 "Download tree has depth %u\n",
2307 dc->top = GNUNET_FS_make_top (dc->h,
2308 &GNUNET_FS_download_signal_suspend_,
2311 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2318 * Download parts of a file based on a search result. The download
2319 * will be associated with the search result (and the association
2320 * will be preserved when serializing/deserializing the state).
2321 * If the search is stopped, the download will not be aborted but
2322 * be 'promoted' to a stand-alone download.
2324 * As with the other download function, this will store
2325 * the blocks at the respective offset in the given file. Also, the
2326 * download is still using the blocking of the underlying FS
2327 * encoding. As a result, the download may *write* outside of the
2328 * given boundaries (if offset and length do not match the 32k FS
2329 * block boundaries). <p>
2331 * The given range can be used to focus a download towards a
2332 * particular portion of the file (optimization), not to strictly
2333 * limit the download to exactly those bytes.
2335 * @param h handle to the file sharing subsystem
2336 * @param sr the search result to use for the download (determines uri and
2337 * meta data and associations)
2338 * @param filename where to store the file, maybe NULL (then no file is
2339 * created on disk and data must be grabbed from the callbacks)
2340 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2341 * can be NULL (in which case we will pick a name if needed); the temporary file
2342 * may already exist, in which case we will try to use the data that is there and
2343 * if it is not what is desired, will overwrite it
2344 * @param offset at what offset should we start the download (typically 0)
2345 * @param length how many bytes should be downloaded starting at offset
2346 * @param anonymity anonymity level to use for the download
2347 * @param options various download options
2348 * @param cctx initial value for the client context for this download
2349 * @return context that can be used to control this download
2351 struct GNUNET_FS_DownloadContext *
2352 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
2353 struct GNUNET_FS_SearchResult *sr,
2354 const char *filename,
2355 const char *tempname,
2359 enum GNUNET_FS_DownloadOptions options,
2362 struct GNUNET_FS_DownloadContext *dc;
2364 if ( (sr == NULL) ||
2365 (sr->download != NULL) )
2370 GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri) ||
2371 GNUNET_FS_uri_test_loc (sr->uri) );
2372 if ( (offset + length < offset) ||
2373 (offset + length > sr->uri->data.chk.file_length) )
2379 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2380 "Starting download `%s' of %llu bytes\n",
2382 (unsigned long long) length);
2384 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2388 if (sr->probe_ctx != NULL)
2390 GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
2391 sr->probe_ctx = NULL;
2393 dc->uri = GNUNET_FS_uri_dup (sr->uri);
2394 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
2395 dc->client_info = cctx;
2396 dc->start_time = GNUNET_TIME_absolute_get ();
2397 if (NULL != filename)
2399 dc->filename = GNUNET_strdup (filename);
2400 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2401 GNUNET_DISK_file_size (filename,
2405 if (GNUNET_FS_uri_test_loc (dc->uri))
2406 GNUNET_assert (GNUNET_OK ==
2407 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2409 dc->offset = offset;
2410 dc->length = length;
2411 dc->anonymity = anonymity;
2412 dc->options = options;
2413 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2414 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
2415 if ( (filename == NULL) &&
2416 (is_recursive_download (dc) ) )
2418 if (tempname != NULL)
2419 dc->temp_filename = GNUNET_strdup (tempname);
2421 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2425 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2426 "Download tree has depth %u\n",
2429 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2436 * Start the downloading process (by entering the queue).
2438 * @param dc our download context
2441 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
2443 if (dc->completed == dc->length)
2445 GNUNET_assert (dc->job_queue == NULL);
2446 dc->job_queue = GNUNET_FS_queue_ (dc->h,
2447 &activate_fs_download,
2448 &deactivate_fs_download,
2450 (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
2455 * Stop a download (aborts if download is incomplete).
2457 * @param dc handle for the download
2458 * @param do_delete delete files of incomplete downloads
2461 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
2464 struct GNUNET_FS_ProgressInfo pi;
2467 if (dc->top != NULL)
2468 GNUNET_FS_end_top (dc->h, dc->top);
2471 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2473 GNUNET_SCHEDULER_cancel (dc->task);
2474 dc->task = GNUNET_SCHEDULER_NO_TASK;
2476 if (dc->search != NULL)
2478 dc->search->download = NULL;
2481 if (dc->job_queue != NULL)
2483 GNUNET_FS_dequeue_ (dc->job_queue);
2484 dc->job_queue = NULL;
2488 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2491 have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
2492 while (NULL != dc->child_head)
2493 GNUNET_FS_download_stop (dc->child_head,
2495 if (dc->parent != NULL)
2496 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2497 dc->parent->child_tail,
2499 if (dc->serialization != NULL)
2500 GNUNET_FS_remove_sync_file_ (dc->h,
2501 ( (dc->parent != NULL) || (dc->search != NULL) )
2502 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2503 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
2505 if ( (GNUNET_YES == have_children) &&
2506 (dc->parent == NULL) )
2507 GNUNET_FS_remove_sync_dir_ (dc->h,
2508 (dc->search != NULL)
2509 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2510 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
2512 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
2513 GNUNET_FS_download_make_status_ (&pi, dc);
2514 GNUNET_FS_free_download_request_ (dc->top_request);
2515 dc->top_request = NULL;
2516 if (dc->active != NULL)
2518 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2521 if (dc->filename != NULL)
2523 if ( (dc->completed != dc->length) &&
2524 (GNUNET_YES == do_delete) )
2526 if (0 != UNLINK (dc->filename))
2527 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
2531 GNUNET_free (dc->filename);
2533 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2534 GNUNET_FS_uri_destroy (dc->uri);
2535 if (NULL != dc->temp_filename)
2537 if (0 != UNLINK (dc->temp_filename))
2538 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
2541 GNUNET_free (dc->temp_filename);
2543 GNUNET_free_non_null (dc->serialization);
2547 /* end of fs_download.c */