2 This file is part of GNUnet.
3 (C) 2001-2011 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - different priority for scheduling probe downloads?
29 #include "gnunet_constants.h"
30 #include "gnunet_fs_service.h"
34 #define DEBUG_DOWNLOAD GNUNET_NO
37 * Determine if the given download (options and meta data) should cause
38 * use to try to do a recursive download.
41 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
43 return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
44 ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
45 ( (dc->meta == NULL) &&
46 ( (NULL == dc->filename) ||
47 ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
49 strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
50 GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
55 * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
56 * only have to truncate the file once we're done).
58 * Given the offset of a block (with respect to the DBLOCKS) and its
59 * depth, return the offset where we would store this block in the
62 * @param fsize overall file size
63 * @param off offset of the block in the file
64 * @param depth depth of the block in the tree, 0 for DBLOCK
65 * @return off for DBLOCKS (depth == treedepth),
66 * otherwise an offset past the end
67 * of the file that does not overlap
68 * with the range for any other block
71 compute_disk_offset (uint64_t fsize,
76 uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
77 uint64_t loff; /* where do IBlocks for depth "i" start? */
78 unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
82 /* first IBlocks start at the end of file, rounded up
83 to full DBLOCK_SIZE */
84 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
85 lsize = ( (fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
86 GNUNET_assert (0 == (off % DBLOCK_SIZE));
87 ioff = (off / DBLOCK_SIZE);
91 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
92 GNUNET_assert (lsize > 0);
93 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
94 ioff /= CHK_PER_INODE;
96 return loff + ioff * sizeof (struct ContentHashKey);
101 * Given a block at the given offset and depth, calculate the offset
102 * for the CHK at the given index.
104 * @param offset the offset of the first
105 * DBLOCK in the subtree of the
107 * @param depth the depth of the IBLOCK in the tree, 0 for DBLOCK
108 * @param k which CHK in the IBLOCK are we
110 * @return offset if k=0, otherwise an appropriately
111 * larger value (i.e., if depth = 1,
112 * the returned value should be offset+k*DBLOCK_SIZE)
115 compute_dblock_offset (uint64_t offset,
120 uint64_t lsize; /* what is the size of the sum of all DBlocks
121 that a CHK at depth i corresponds to? */
126 for (i=1;i<depth;i++)
127 lsize *= CHK_PER_INODE;
128 return offset + k * lsize;
133 * Fill in all of the generic fields for a download event and call the
136 * @param pi structure to fill in
137 * @param dc overall download context
140 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
141 struct GNUNET_FS_DownloadContext *dc)
143 pi->value.download.dc = dc;
144 pi->value.download.cctx
146 pi->value.download.pctx
147 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
148 pi->value.download.sctx
149 = (dc->search == NULL) ? NULL : dc->search->client_info;
150 pi->value.download.uri
152 pi->value.download.filename
154 pi->value.download.size
156 pi->value.download.duration
157 = GNUNET_TIME_absolute_get_duration (dc->start_time);
158 pi->value.download.completed
160 pi->value.download.anonymity
162 pi->value.download.eta
163 = GNUNET_TIME_calculate_eta (dc->start_time,
166 pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
167 if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
168 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
171 dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
177 * We're ready to transmit a search request to the
178 * file-sharing service. Do it. If there is
179 * more than one request pending, try to send
180 * multiple or request another transmission.
183 * @param size number of bytes available in buf
184 * @param buf where the callee should write the message
185 * @return number of bytes written to buf
188 transmit_download_request (void *cls,
194 * Closure for iterator processing results.
196 struct ProcessResultClosure
202 GNUNET_HashCode query;
205 * Data found in P2P network.
210 * Our download context.
212 struct GNUNET_FS_DownloadContext *dc;
215 * Number of bytes in data.
222 enum GNUNET_BLOCK_Type type;
225 * Flag to indicate if this block should be stored on disk.
233 * Iterator over entries in the pending requests in the 'active' map for the
234 * reply that we just got.
236 * @param cls closure (our 'struct ProcessResultClosure')
237 * @param key query for the given value / request
238 * @param value value in the hash map (a 'struct DownloadRequest')
239 * @return GNUNET_YES (we should continue to iterate); unless serious error
242 process_result_with_request (void *cls,
243 const GNUNET_HashCode * key,
248 * We've found a matching block without downloading it.
249 * Encrypt it and pass it to our "receive" function as
250 * if we had received it from the network.
252 * @param dc download in question
253 * @param chk request this relates to
254 * @param dr request details
255 * @param block plaintext data matching request
256 * @param len number of bytes in block
257 * @param do_store should we still store the block on disk?
258 * @return GNUNET_OK on success
261 encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc,
262 const struct ContentHashKey *chk,
263 struct DownloadRequest *dr,
268 struct ProcessResultClosure prc;
270 struct GNUNET_CRYPTO_AesSessionKey sk;
271 struct GNUNET_CRYPTO_AesInitializationVector iv;
272 GNUNET_HashCode query;
274 GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
275 if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
281 return GNUNET_SYSERR;
283 GNUNET_CRYPTO_hash (enc, len, &query);
284 if (0 != memcmp (&query,
286 sizeof (GNUNET_HashCode)))
289 return GNUNET_SYSERR;
292 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
293 "Matching block already present, no need for download!\n");
295 /* already got it! */
299 prc.type = (0 == dr->depth)
300 ? GNUNET_BLOCK_TYPE_FS_DBLOCK
301 : GNUNET_BLOCK_TYPE_FS_IBLOCK;
302 prc.query = chk->query;
303 prc.do_store = do_store;
304 process_result_with_request (&prc,
312 * We've lost our connection with the FS service.
313 * Re-establish it and re-transmit all of our
316 * @param dc download context that is having trouble
319 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
323 * We found an entry in a directory. Check if the respective child
324 * already exists and if not create the respective child download.
326 * @param cls the parent download
327 * @param filename name of the file in the directory
328 * @param uri URI of the file (CHK or LOC)
329 * @param meta meta data of the file
330 * @param length number of bytes in data
331 * @param data contents of the file (or NULL if they were not inlined)
334 trigger_recursive_download (void *cls,
335 const char *filename,
336 const struct GNUNET_FS_Uri *uri,
337 const struct GNUNET_CONTAINER_MetaData *meta,
343 * We're done downloading a directory. Open the file and
344 * trigger all of the (remaining) child downloads.
346 * @param dc context of download that just completed
349 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
354 struct GNUNET_DISK_FileHandle *h;
355 struct GNUNET_DISK_MapHandle *m;
357 size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
358 size = (size_t) size64;
359 if (size64 != (uint64_t) size)
361 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
362 _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
365 if (dc->filename != NULL)
367 h = GNUNET_DISK_file_open (dc->filename,
368 GNUNET_DISK_OPEN_READ,
369 GNUNET_DISK_PERM_NONE);
373 GNUNET_assert (dc->temp_filename != NULL);
374 h = GNUNET_DISK_file_open (dc->temp_filename,
375 GNUNET_DISK_OPEN_READ,
376 GNUNET_DISK_PERM_NONE);
380 data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
383 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
384 _("Directory too large for system address space\n"));
388 GNUNET_FS_directory_list_contents (size,
391 &trigger_recursive_download,
393 GNUNET_DISK_file_unmap (m);
395 GNUNET_DISK_file_close (h);
396 if (dc->filename == NULL)
398 if (0 != UNLINK (dc->temp_filename))
399 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
402 GNUNET_free (dc->temp_filename);
403 dc->temp_filename = NULL;
409 * Check if all child-downloads have completed (or trigger them if
410 * necessary) and once we're completely done, signal completion (and
411 * possibly recurse to parent). This function MUST be called when the
412 * download of a file itself is done or when the download of a file is
413 * done and then later a direct child download has completed (and
414 * hence this download may complete itself).
416 * @param dc download to check for completion of children
419 check_completed (struct GNUNET_FS_DownloadContext *dc)
421 struct GNUNET_FS_ProgressInfo pi;
422 struct GNUNET_FS_DownloadContext *pos;
424 /* first, check if we need to download children */
425 if ( (dc->child_head == NULL) &&
426 (is_recursive_download (dc)) )
427 full_recursive_download (dc);
428 /* then, check if children are done already */
429 pos = dc->child_head;
432 if ( (pos->emsg == NULL) &&
433 (pos->completed < pos->length) )
434 return; /* not done yet */
435 if ( (pos->child_head != NULL) &&
436 (pos->has_finished != GNUNET_YES) )
437 return; /* not transitively done yet */
440 /* All of our children are done, so mark this download done */
441 dc->has_finished = GNUNET_YES;
442 GNUNET_FS_download_sync_ (dc);
444 /* signal completion */
445 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
446 GNUNET_FS_download_make_status_ (&pi, dc);
448 /* let parent know */
449 if (dc->parent != NULL)
450 check_completed (dc->parent);
455 * We got a block of plaintext data (from the meta data).
456 * Try it for upward reconstruction of the data. On success,
457 * the top-level block will move to state BRS_DOWNLOAD_UP.
459 * @param dr one of our request entries
460 * @param data plaintext data, starting from the beginning of the file
461 * @param data_len number of bytes in data
464 try_match_block (struct GNUNET_FS_DownloadContext *dc,
465 struct DownloadRequest *dr,
469 struct GNUNET_FS_ProgressInfo pi;
471 char enc[DBLOCK_SIZE];
472 struct ContentHashKey chks[CHK_PER_INODE];
473 struct ContentHashKey chk;
474 struct GNUNET_CRYPTO_AesSessionKey sk;
475 struct GNUNET_CRYPTO_AesInitializationVector iv;
477 struct DownloadRequest *drc;
478 struct GNUNET_DISK_FileHandle *fh;
482 if (BRS_DOWNLOAD_UP == dr->state)
486 complete = GNUNET_YES;
487 for (i=0;i<dr->num_children;i++)
489 drc = dr->children[i];
493 if (drc->state != BRS_RECONSTRUCT_META_UP)
494 complete = GNUNET_NO;
496 if (GNUNET_YES != complete)
498 data = (const char*) chks;
499 dlen = dr->num_children * sizeof (struct ContentHashKey);
503 if (dr->offset > data_len)
505 dlen = GNUNET_MIN (data_len - dr->offset,
508 GNUNET_CRYPTO_hash (&data[dr->offset],
511 GNUNET_CRYPTO_hash_to_aes_key (&chk.key, &sk, &iv);
512 if (-1 == GNUNET_CRYPTO_aes_encrypt (data, dlen,
520 GNUNET_CRYPTO_hash (enc, dlen, &chk.query);
525 dr->state = BRS_RECONSTRUCT_META_UP;
528 if (0 != memcmp (&chk,
530 sizeof (struct ContentHashKey)))
532 /* other peer provided bogus meta data */
536 /* write block to disk */
537 fn = dc->filename != NULL
540 fh = GNUNET_DISK_file_open (fn,
541 GNUNET_DISK_OPEN_READWRITE |
542 GNUNET_DISK_OPEN_CREATE |
543 GNUNET_DISK_OPEN_TRUNCATE,
544 GNUNET_DISK_PERM_USER_READ |
545 GNUNET_DISK_PERM_USER_WRITE |
546 GNUNET_DISK_PERM_GROUP_READ |
547 GNUNET_DISK_PERM_OTHER_READ);
550 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
553 GNUNET_asprintf (&dc->emsg,
554 _("Failed to open file `%s' for writing"),
556 GNUNET_DISK_file_close (fh);
557 dr->state = BRS_ERROR;
558 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
559 pi.value.download.specifics.error.message = dc->emsg;
560 GNUNET_FS_download_make_status_ (&pi, dc);
564 GNUNET_DISK_file_write (fh,
568 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
571 GNUNET_asprintf (&dc->emsg,
572 _("Failed to open file `%s' for writing"),
574 GNUNET_DISK_file_close (fh);
575 dr->state = BRS_ERROR;
576 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
577 pi.value.download.specifics.error.message = dc->emsg;
578 GNUNET_FS_download_make_status_ (&pi, dc);
581 GNUNET_DISK_file_close (fh);
583 dr->state = BRS_DOWNLOAD_UP;
584 dc->completed = dc->length;
585 GNUNET_FS_download_sync_ (dc);
586 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
587 pi.value.download.specifics.progress.data = data;
588 pi.value.download.specifics.progress.offset = 0;
589 pi.value.download.specifics.progress.data_len = dlen;
590 pi.value.download.specifics.progress.depth = 0;
591 GNUNET_FS_download_make_status_ (&pi, dc);
592 check_completed (dc);
595 /* how did we get here? */
603 * Type of a function that libextractor calls for each
604 * meta data item found. If we find full data meta data,
605 * call 'try_match_block' on it.
607 * @param cls our 'struct GNUNET_FS_DownloadContext*'
608 * @param plugin_name name of the plugin that produced this value;
609 * special values can be used (i.e. '<zlib>' for zlib being
610 * used in the main libextractor library and yielding
612 * @param type libextractor-type describing the meta data
613 * @param format basic format information about data
614 * @param data_mime_type mime-type of data (not of the original file);
615 * can be NULL (if mime-type is not known)
616 * @param data actual meta-data found
617 * @param data_len number of bytes in data
618 * @return 0 to continue extracting, 1 to abort
621 match_full_data (void *cls,
622 const char *plugin_name,
623 enum EXTRACTOR_MetaType type,
624 enum EXTRACTOR_MetaFormat format,
625 const char *data_mime_type,
629 struct GNUNET_FS_DownloadContext *dc = cls;
631 if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
633 if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
636 return 1; /* bogus meta data */
647 * Set the state of the given download request to
648 * BRS_DOWNLOAD_UP and propagate it up the tree.
650 * @param dr download request that is done
653 propagate_up (struct DownloadRequest *dr)
659 dr->state = BRS_DOWNLOAD_UP;
663 for (i=0;i<dr->num_children;i++)
664 if (dr->children[i]->state != BRS_DOWNLOAD_UP)
667 while (i == dr->num_children);
672 * Try top-down reconstruction. Before, the given request node
673 * must have the state BRS_CHK_SET. Afterwards, more nodes may
674 * have that state or advanced to BRS_DOWNLOAD_DOWN or even
675 * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the
678 * @param dc overall download this block belongs to
679 * @param dr block to reconstruct
682 try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
683 struct DownloadRequest *dr)
686 char block[DBLOCK_SIZE];
691 unsigned int chk_off;
692 struct DownloadRequest *drc;
693 uint64_t child_block_size;
694 const struct ContentHashKey *chks;
697 GNUNET_assert (dc->rfh != NULL);
698 GNUNET_assert (dr->state == BRS_CHK_SET);
699 total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
700 GNUNET_assert (dr->depth < dc->treedepth);
701 len = GNUNET_FS_tree_calculate_block_size (total,
704 GNUNET_assert (len <= DBLOCK_SIZE);
705 off = compute_disk_offset (total,
708 if (dc->old_file_size < off + len)
709 return; /* failure */
711 GNUNET_DISK_file_seek (dc->rfh,
713 GNUNET_DISK_SEEK_SET) )
715 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
718 return; /* failure */
721 GNUNET_DISK_file_read (dc->rfh,
725 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
728 return; /* failure */
730 GNUNET_CRYPTO_hash (block, len, &key);
731 if (0 != memcmp (&key,
733 sizeof (GNUNET_HashCode)))
734 return; /* mismatch */
736 encrypt_existing_match (dc,
743 /* hash matches but encrypted block does not, really bad */
744 dr->state = BRS_ERROR;
746 while (dr->parent != NULL)
749 dr->state = BRS_ERROR;
754 dr->state = BRS_DOWNLOAD_DOWN;
756 /* set CHKs for children */
757 up_done = GNUNET_YES;
758 chks = (const struct ContentHashKey*) block;
759 for (i=0;i<dr->num_children;i++)
761 drc = dr->children[i];
762 GNUNET_assert (drc->offset >= dr->offset);
763 child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
764 GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
765 chk_off = (drc->offset - dr->offset) / child_block_size;
766 GNUNET_assert (drc->state == BRS_INIT);
767 drc->state = BRS_CHK_SET;
768 drc->chk = chks[chk_off];
769 try_top_down_reconstruction (dc, drc);
770 if (drc->state != BRS_DOWNLOAD_UP)
771 up_done = GNUNET_NO; /* children not all done */
773 if (up_done == GNUNET_YES)
774 propagate_up (dr); /* children all done (or no children...) */
779 * Schedule the download of the specified block in the tree.
781 * @param dc overall download this block belongs to
782 * @param chk content-hash-key of the block
783 * @param offset offset of the block in the file
784 * (for IBlocks, the offset is the lowest
785 * offset of any DBlock in the subtree under
787 * @param depth depth of the block, 0 is the root of the tree
790 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
791 struct DownloadRequest *dr)
800 case BRS_RECONSTRUCT_DOWN:
803 case BRS_RECONSTRUCT_META_UP:
806 case BRS_RECONSTRUCT_UP:
810 /* normal case, start download */
812 case BRS_DOWNLOAD_DOWN:
813 for (i=0;i<dr->num_children;i++)
814 schedule_block_download (dc, dr->children[i]);
816 case BRS_DOWNLOAD_UP:
824 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
825 "Scheduling download at offset %llu and depth %u for `%s'\n",
826 (unsigned long long) dr->offset,
828 GNUNET_h2s (&dr->chk.query));
830 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
833 dr->is_pending = GNUNET_YES;
834 GNUNET_CONTAINER_multihashmap_put (dc->active,
837 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
838 if (dc->client == NULL)
839 return; /* download not active */
841 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
842 sizeof (struct SearchMessage),
843 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
845 &transmit_download_request,
850 #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
853 * We found an entry in a directory. Check if the respective child
854 * already exists and if not create the respective child download.
856 * @param cls the parent download
857 * @param filename name of the file in the directory
858 * @param uri URI of the file (CHK or LOC)
859 * @param meta meta data of the file
860 * @param length number of bytes in data
861 * @param data contents of the file (or NULL if they were not inlined)
864 trigger_recursive_download (void *cls,
865 const char *filename,
866 const struct GNUNET_FS_Uri *uri,
867 const struct GNUNET_CONTAINER_MetaData *meta,
871 struct GNUNET_FS_DownloadContext *dc = cls;
872 struct GNUNET_FS_DownloadContext *cpos;
873 struct GNUNET_DISK_FileHandle *fh;
875 const char *real_name;
884 return; /* entry for the directory itself */
885 cpos = dc->child_head;
888 if ( (GNUNET_FS_uri_test_equal (uri,
890 ( (filename != NULL) &&
891 (0 == strcmp (cpos->filename,
897 return; /* already exists */
899 if (NULL == filename)
901 fn = GNUNET_FS_meta_data_suggest_filename (meta);
904 us = GNUNET_FS_uri_to_string (uri);
905 fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_CHK_PREFIX)]);
908 else if (fn[0] == '.')
911 us = GNUNET_FS_uri_to_string (uri);
912 GNUNET_asprintf (&fn,
914 &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], ext);
918 /* change '\' to '/' (this should have happened
919 during insertion, but malicious peers may
920 not have done this) */
921 while (NULL != (pos = strstr (fn, "\\")))
923 /* remove '../' everywhere (again, well-behaved
924 peers don't do this, but don't trust that
925 we did not get something nasty) */
926 while (NULL != (pos = strstr (fn, "../")))
934 if (dc->filename == NULL)
940 dn = GNUNET_strdup (dc->filename);
941 GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
943 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
944 GNUNET_FS_DIRECTORY_EXT)) );
945 if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
947 strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
948 GNUNET_FS_DIRECTORY_EXT)) )
949 dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
950 if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
951 ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
953 strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
954 GNUNET_FS_DIRECTORY_EXT)) ) )
956 GNUNET_asprintf (&full_name,
961 GNUNET_FS_DIRECTORY_EXT);
965 GNUNET_asprintf (&full_name,
973 if ( (full_name != NULL) &&
975 GNUNET_DISK_directory_create_for_file (full_name)) )
977 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
978 _("Failed to create directory for recursive download of `%s'\n"),
980 GNUNET_free (full_name);
981 GNUNET_free_non_null (fn);
986 if ( (data != NULL) &&
987 (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
989 if (full_name == NULL)
991 temp_name = GNUNET_DISK_mktemp ("gnunet-download-trd");
992 real_name = temp_name;
996 real_name = full_name;
998 /* write to disk, then trigger normal download which will instantly progress to completion */
999 fh = GNUNET_DISK_file_open (real_name,
1000 GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
1001 GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
1004 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1007 GNUNET_free (full_name);
1008 GNUNET_free_non_null (fn);
1012 GNUNET_DISK_file_write (fh,
1016 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1020 GNUNET_DISK_file_close (fh);
1022 GNUNET_FS_download_start (dc->h,
1025 full_name, temp_name,
1027 GNUNET_FS_uri_chk_get_file_size (uri),
1032 GNUNET_free_non_null (full_name);
1033 GNUNET_free_non_null (temp_name);
1034 GNUNET_free_non_null (fn);
1039 * (recursively) free download request structure
1041 * @param dr request to free
1044 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
1050 for (i=0;i<dr->num_children;i++)
1051 GNUNET_FS_free_download_request_ (dr->children[i]);
1052 GNUNET_free_non_null (dr->children);
1058 * Iterator over entries in the pending requests in the 'active' map for the
1059 * reply that we just got.
1061 * @param cls closure (our 'struct ProcessResultClosure')
1062 * @param key query for the given value / request
1063 * @param value value in the hash map (a 'struct DownloadRequest')
1064 * @return GNUNET_YES (we should continue to iterate); unless serious error
1067 process_result_with_request (void *cls,
1068 const GNUNET_HashCode *key,
1071 struct ProcessResultClosure *prc = cls;
1072 struct DownloadRequest *dr = value;
1073 struct GNUNET_FS_DownloadContext *dc = prc->dc;
1074 struct DownloadRequest *drc;
1075 struct GNUNET_DISK_FileHandle *fh = NULL;
1076 struct GNUNET_CRYPTO_AesSessionKey skey;
1077 struct GNUNET_CRYPTO_AesInitializationVector iv;
1079 struct GNUNET_FS_ProgressInfo pi;
1084 struct ContentHashKey *chk;
1087 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1088 "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n",
1091 (unsigned long long) dr->offset,
1092 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1095 bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
1098 if (prc->size != bs)
1100 GNUNET_asprintf (&dc->emsg,
1101 _("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"),
1104 (unsigned long long) dr->offset,
1105 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
1107 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1110 while (dr->parent != NULL)
1112 dr->state = BRS_ERROR;
1115 dr->state = BRS_ERROR;
1119 GNUNET_assert (GNUNET_YES ==
1120 GNUNET_CONTAINER_multihashmap_remove (dc->active,
1123 if (GNUNET_YES == dr->is_pending)
1125 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1128 dr->is_pending = GNUNET_NO;
1132 GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
1133 if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
1140 dc->emsg = GNUNET_strdup (_("internal error decrypting content"));
1143 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
1147 if ( ( GNUNET_YES == prc->do_store) &&
1148 ( (dc->filename != NULL) ||
1149 (is_recursive_download (dc)) ) &&
1150 ( (dr->depth == dc->treedepth) ||
1151 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
1153 fh = GNUNET_DISK_file_open (dc->filename != NULL
1155 : dc->temp_filename,
1156 GNUNET_DISK_OPEN_READWRITE |
1157 GNUNET_DISK_OPEN_CREATE,
1158 GNUNET_DISK_PERM_USER_READ |
1159 GNUNET_DISK_PERM_USER_WRITE |
1160 GNUNET_DISK_PERM_GROUP_READ |
1161 GNUNET_DISK_PERM_OTHER_READ);
1164 GNUNET_asprintf (&dc->emsg,
1165 _("Download failed: could not open file `%s': %s\n"),
1171 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1172 "Saving decrypted block to disk at offset %llu\n",
1173 (unsigned long long) off);
1176 GNUNET_DISK_file_seek (fh,
1178 GNUNET_DISK_SEEK_SET) ) )
1180 GNUNET_asprintf (&dc->emsg,
1181 _("Failed to seek to offset %llu in file `%s': %s\n"),
1182 (unsigned long long) off,
1188 GNUNET_DISK_file_write (fh,
1192 GNUNET_asprintf (&dc->emsg,
1193 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1194 (unsigned int) prc->size,
1195 (unsigned long long) off,
1200 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1206 /* DBLOCK, update progress and try recursion if applicable */
1208 if (dr->offset < dc->offset)
1210 /* starting offset begins in the middle of pt,
1211 do not count first bytes as progress */
1212 GNUNET_assert (app > (dc->offset - dr->offset));
1213 app -= (dc->offset - dr->offset);
1215 if (dr->offset + prc->size > dc->offset + dc->length)
1217 /* end of block is after relevant range,
1218 do not count last bytes as progress */
1219 GNUNET_assert (app > (dr->offset + prc->size) - (dc->offset + dc->length));
1220 app -= (dr->offset + prc->size) - (dc->offset + dc->length);
1222 dc->completed += app;
1224 /* do recursive download if option is set and either meta data
1225 says it is a directory or if no meta data is given AND filename
1226 ends in '.gnd' (top-level case) */
1227 if (is_recursive_download (dc))
1228 GNUNET_FS_directory_list_contents (prc->size,
1231 &trigger_recursive_download,
1235 dr->state = BRS_DOWNLOAD_DOWN;
1236 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1237 pi.value.download.specifics.progress.data = pt;
1238 pi.value.download.specifics.progress.offset = dr->offset;
1239 pi.value.download.specifics.progress.data_len = prc->size;
1240 pi.value.download.specifics.progress.depth = dr->depth;
1241 GNUNET_FS_download_make_status_ (&pi, dc);
1242 GNUNET_assert (dc->completed <= dc->length);
1244 if (dc->completed == dc->length)
1246 /* download completed, signal */
1248 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1249 "Download completed, truncating file to desired length %llu\n",
1250 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1252 /* truncate file to size (since we store IBlocks at the end) */
1253 if (dc->filename != NULL)
1255 if (0 != truncate (dc->filename,
1256 GNUNET_ntohll (dc->uri->data.chk.file_length)))
1257 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1261 if (dc->job_queue != NULL)
1263 GNUNET_FS_dequeue_ (dc->job_queue);
1264 dc->job_queue = NULL;
1266 GNUNET_assert (dr->depth == 0);
1267 check_completed (dc);
1272 /* bottom of the tree, no child downloads possible, just sync */
1273 GNUNET_FS_download_sync_ (dc);
1278 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1279 "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1281 (unsigned long long) dr->offset);
1283 GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1284 chk = (struct ContentHashKey*) pt;
1285 for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1287 off = compute_dblock_offset (dr->offset,
1290 drc = dr->children[i];
1295 drc->state = BRS_CHK_SET;
1296 schedule_block_download (dc, drc);
1298 case BRS_RECONSTRUCT_DOWN:
1301 case BRS_RECONSTRUCT_META_UP:
1304 case BRS_RECONSTRUCT_UP:
1310 case BRS_DOWNLOAD_DOWN:
1313 case BRS_DOWNLOAD_UP:
1324 GNUNET_FS_download_sync_ (dc);
1329 GNUNET_DISK_file_close (fh);
1330 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1331 pi.value.download.specifics.error.message = dc->emsg;
1332 GNUNET_FS_download_make_status_ (&pi, dc);
1333 /* abort all pending requests */
1336 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1339 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1341 GNUNET_FS_free_download_request_ (dc->top_request);
1342 dc->top_request = NULL;
1343 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1345 dc->pending_head = NULL;
1346 dc->pending_tail = NULL;
1347 GNUNET_FS_download_sync_ (dc);
1353 * Process a download result.
1355 * @param dc our download context
1356 * @param type type of the result
1357 * @param data the (encrypted) response
1358 * @param size size of data
1361 process_result (struct GNUNET_FS_DownloadContext *dc,
1362 enum GNUNET_BLOCK_Type type,
1366 struct ProcessResultClosure prc;
1372 prc.do_store = GNUNET_YES;
1373 GNUNET_CRYPTO_hash (data, size, &prc.query);
1375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1376 "Received result for query `%s' from `%s'-service\n",
1377 GNUNET_h2s (&prc.query),
1380 GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1382 &process_result_with_request,
1388 * Type of a function to call when we receive a message
1391 * @param cls closure
1392 * @param msg message received, NULL on timeout or fatal error
1395 receive_results (void *cls,
1396 const struct GNUNET_MessageHeader * msg)
1398 struct GNUNET_FS_DownloadContext *dc = cls;
1399 const struct PutMessage *cm;
1402 if ( (NULL == msg) ||
1403 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1404 (sizeof (struct PutMessage) > ntohs(msg->size)) )
1406 GNUNET_break (msg == NULL);
1410 msize = ntohs(msg->size);
1411 cm = (const struct PutMessage*) msg;
1415 msize - sizeof (struct PutMessage));
1416 if (dc->client == NULL)
1417 return; /* fatal error */
1418 /* continue receiving */
1419 GNUNET_CLIENT_receive (dc->client,
1422 GNUNET_TIME_UNIT_FOREVER_REL);
1428 * We're ready to transmit a search request to the
1429 * file-sharing service. Do it. If there is
1430 * more than one request pending, try to send
1431 * multiple or request another transmission.
1433 * @param cls closure
1434 * @param size number of bytes available in buf
1435 * @param buf where the callee should write the message
1436 * @return number of bytes written to buf
1439 transmit_download_request (void *cls,
1443 struct GNUNET_FS_DownloadContext *dc = cls;
1445 struct SearchMessage *sm;
1446 struct DownloadRequest *dr;
1452 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1453 "Transmitting download request failed, trying to reconnect\n");
1458 GNUNET_assert (size >= sizeof (struct SearchMessage));
1461 while ( (NULL != (dr = dc->pending_head)) &&
1462 (size >= msize + sizeof (struct SearchMessage)) )
1465 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1466 "Transmitting download request for `%s' to `%s'-service\n",
1467 GNUNET_h2s (&dr->chk.query),
1470 memset (sm, 0, sizeof (struct SearchMessage));
1471 sm->header.size = htons (sizeof (struct SearchMessage));
1472 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1473 if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1474 sm->options = htonl (1);
1476 sm->options = htonl (0);
1478 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
1480 sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
1481 sm->anonymity_level = htonl (dc->anonymity);
1482 sm->target = dc->target.hashPubKey;
1483 sm->query = dr->chk.query;
1484 GNUNET_CONTAINER_DLL_remove (dc->pending_head,
1487 dr->is_pending = GNUNET_NO;
1488 msize += sizeof (struct SearchMessage);
1491 if (dc->pending_head != NULL)
1493 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1494 sizeof (struct SearchMessage),
1495 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1497 &transmit_download_request,
1499 GNUNET_assert (dc->th != NULL);
1506 * Reconnect to the FS service and transmit our queries NOW.
1508 * @param cls our download context
1512 do_reconnect (void *cls,
1513 const struct GNUNET_SCHEDULER_TaskContext *tc)
1515 struct GNUNET_FS_DownloadContext *dc = cls;
1516 struct GNUNET_CLIENT_Connection *client;
1518 dc->task = GNUNET_SCHEDULER_NO_TASK;
1519 client = GNUNET_CLIENT_connect ("fs",
1523 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1524 "Connecting to `%s'-service failed, will try again.\n",
1529 dc->client = client;
1530 if (dc->pending_head != NULL)
1532 dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1533 sizeof (struct SearchMessage),
1534 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1536 &transmit_download_request,
1538 GNUNET_assert (dc->th != NULL);
1540 GNUNET_CLIENT_receive (client,
1543 GNUNET_TIME_UNIT_FOREVER_REL);
1548 * Add entries to the pending list.
1550 * @param cls our download context
1552 * @param entry entry of type "struct DownloadRequest"
1556 retry_entry (void *cls,
1557 const GNUNET_HashCode *key,
1560 struct GNUNET_FS_DownloadContext *dc = cls;
1561 struct DownloadRequest *dr = entry;
1563 GNUNET_CONTAINER_DLL_insert (dc->pending_head,
1566 dr->is_pending = GNUNET_YES;
1572 * We've lost our connection with the FS service.
1573 * Re-establish it and re-transmit all of our
1576 * @param dc download context that is having trouble
1579 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1582 if (NULL != dc->client)
1585 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1586 "Moving all requests back to pending list\n");
1590 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1593 /* full reset of the pending list */
1594 dc->pending_head = NULL;
1595 dc->pending_tail = NULL;
1596 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1599 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1603 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1604 "Will try to reconnect in 1s\n");
1607 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
1614 * We're allowed to ask the FS service for our blocks. Start the download.
1616 * @param cls the 'struct GNUNET_FS_DownloadContext'
1617 * @param client handle to use for communcation with FS (we must destroy it!)
1620 activate_fs_download (void *cls,
1621 struct GNUNET_CLIENT_Connection *client)
1623 struct GNUNET_FS_DownloadContext *dc = cls;
1624 struct GNUNET_FS_ProgressInfo pi;
1627 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1628 "Download activated\n");
1630 GNUNET_assert (NULL != client);
1631 GNUNET_assert (dc->client == NULL);
1632 GNUNET_assert (dc->th == NULL);
1633 dc->client = client;
1634 GNUNET_CLIENT_receive (client,
1637 GNUNET_TIME_UNIT_FOREVER_REL);
1638 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1639 GNUNET_FS_download_make_status_ (&pi, dc);
1640 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1644 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1645 "Asking for transmission to FS service\n");
1647 if (dc->pending_head != NULL)
1649 dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1650 sizeof (struct SearchMessage),
1651 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1653 &transmit_download_request,
1655 GNUNET_assert (dc->th != NULL);
1661 * We must stop to ask the FS service for our blocks. Pause the download.
1663 * @param cls the 'struct GNUNET_FS_DownloadContext'
1666 deactivate_fs_download (void *cls)
1668 struct GNUNET_FS_DownloadContext *dc = cls;
1669 struct GNUNET_FS_ProgressInfo pi;
1672 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1673 "Download deactivated\n");
1677 GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1680 if (NULL != dc->client)
1682 GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1685 pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1686 GNUNET_FS_download_make_status_ (&pi, dc);
1691 * (recursively) Create a download request structure.
1693 * @param parent parent of the current entry
1694 * @param depth depth of the current entry, 0 are the DBLOCKs,
1695 * top level block is 'dc->treedepth - 1'
1696 * @param dr_offset offset in the original file this block maps to
1697 * (as in, offset of the first byte of the first DBLOCK
1698 * in the subtree rooted in the returned download request tree)
1699 * @param file_start_offset desired starting offset for the download
1700 * in the original file; requesting tree should not contain
1701 * DBLOCKs prior to the file_start_offset
1702 * @param file_length desired number of bytes the user wanted to access
1703 * (from file_start_offset). Resulting tree should not contain
1704 * DBLOCKs after file_start_offset + file_length.
1705 * @return download request tree for the given range of DBLOCKs at
1706 * the specified depth
1708 static struct DownloadRequest *
1709 create_download_request (struct DownloadRequest *parent,
1712 uint64_t file_start_offset,
1713 uint64_t desired_length)
1715 struct DownloadRequest *dr;
1717 unsigned int head_skip;
1718 uint64_t child_block_size;
1720 dr = GNUNET_malloc (sizeof (struct DownloadRequest));
1721 dr->parent = parent;
1723 dr->offset = dr_offset;
1726 child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
1728 /* calculate how many blocks at this level are not interesting
1729 from the start (rounded down), either because of the requested
1730 file offset or because this IBlock is further along */
1731 if (dr_offset < file_start_offset)
1732 head_skip = file_start_offset / child_block_size;
1734 head_skip = dr_offset / child_block_size;
1736 /* calculate index of last block at this level that is interesting (rounded up) */
1737 dr->num_children = file_start_offset + desired_length / child_block_size;
1738 if (dr->num_children * child_block_size < file_start_offset + desired_length)
1739 dr->num_children++; /* round up */
1741 /* now we can get the total number of children for this block */
1742 dr->num_children -= head_skip;
1743 if (dr->num_children > CHK_PER_INODE)
1744 dr->num_children = CHK_PER_INODE; /* cap at max */
1746 /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
1747 GNUNET_assert (dr->num_children > 0);
1749 dr->children = GNUNET_malloc (dr->num_children *
1750 sizeof (struct DownloadRequest *));
1751 for (i=0;i<dr->num_children;i++)
1752 dr->children[i] = create_download_request (dr,
1754 dr_offset + i * child_block_size,
1763 * Continuation after a possible attempt to reconstruct
1764 * the current IBlock from the existing file.
1766 * @param cls the 'struct ReconstructContext'
1767 * @param tc scheduler context
1770 reconstruct_cont (void *cls,
1771 const struct GNUNET_SCHEDULER_TaskContext *tc)
1773 struct GNUNET_FS_DownloadContext *dc = cls;
1775 /* clean up state from tree encoder */
1778 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
1781 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
1783 GNUNET_SCHEDULER_cancel (dc->task);
1784 dc->task = GNUNET_SCHEDULER_NO_TASK;
1786 if (dc->rfh != NULL)
1788 GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
1791 /* start "normal" download */
1792 schedule_block_download (dc,
1798 * Task requesting the next block from the tree encoder.
1800 * @param tc task context
1803 get_next_block (void *cls,
1804 const struct GNUNET_SCHEDULER_TaskContext *tc)
1806 struct GNUNET_FS_DownloadContext *dc = cls;
1808 dc->task = GNUNET_SCHEDULER_NO_TASK;
1809 GNUNET_FS_tree_encoder_next (dc->te);
1815 * Function called asking for the current (encoded)
1816 * block to be processed. After processing the
1817 * client should either call "GNUNET_FS_tree_encode_next"
1818 * or (on error) "GNUNET_FS_tree_encode_finish".
1820 * This function checks if the content on disk matches
1821 * the expected content based on the URI.
1823 * @param cls closure
1824 * @param chk content hash key for the block
1825 * @param offset offset of the block
1826 * @param depth depth of the block, 0 for DBLOCK
1827 * @param type type of the block (IBLOCK or DBLOCK)
1828 * @param block the (encrypted) block
1829 * @param block_size size of block (in bytes)
1832 reconstruct_cb (void *cls,
1833 const struct ContentHashKey *chk,
1836 enum GNUNET_BLOCK_Type type,
1838 uint16_t block_size)
1840 struct GNUNET_FS_DownloadContext *dc = cls;
1841 struct GNUNET_FS_ProgressInfo pi;
1842 struct DownloadRequest *dr;
1846 /* find corresponding request entry */
1847 dr = dc->top_request;
1848 while (dr->depth > depth)
1850 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1851 chld = (offset - dr->offset) / blen;
1852 GNUNET_assert (chld < dr->num_children);
1853 dr = dr->children[chld];
1859 case BRS_RECONSTRUCT_DOWN:
1861 case BRS_RECONSTRUCT_META_UP:
1863 case BRS_RECONSTRUCT_UP:
1866 if (0 == memcmp (chk,
1868 sizeof (struct ContentHashKey)))
1870 /* block matches, hence tree below matches;
1871 this request is done! */
1872 dr->state = BRS_DOWNLOAD_UP;
1873 /* calculate how many bytes of payload this block
1875 blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
1876 /* how many of those bytes are in the requested range? */
1877 blen = GNUNET_MIN (blen,
1878 dc->length + dc->offset - dr->offset);
1879 /* signal progress */
1880 dc->completed += blen;
1881 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1882 pi.value.download.specifics.progress.data = NULL;
1883 pi.value.download.specifics.progress.offset = offset;
1884 pi.value.download.specifics.progress.data_len = 0;
1885 pi.value.download.specifics.progress.depth = 0;
1886 GNUNET_FS_download_make_status_ (&pi, dc);
1892 case BRS_DOWNLOAD_DOWN:
1894 case BRS_DOWNLOAD_UP:
1902 if ( (dr == dc->top_request) &&
1903 (dr->state == BRS_DOWNLOAD_UP) )
1905 check_completed (dc);
1908 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
1914 * Function called by the tree encoder to obtain a block of plaintext
1915 * data (for the lowest level of the tree).
1917 * @param cls our 'struct ReconstructContext'
1918 * @param offset identifies which block to get
1919 * @param max (maximum) number of bytes to get; returning
1920 * fewer will also cause errors
1921 * @param buf where to copy the plaintext buffer
1922 * @param emsg location to store an error message (on error)
1923 * @return number of bytes copied to buf, 0 on error
1926 fh_reader (void *cls,
1932 struct GNUNET_FS_DownloadContext *dc = cls;
1933 struct GNUNET_DISK_FileHandle *fh = dc->rfh;
1938 GNUNET_DISK_file_seek (fh,
1940 GNUNET_DISK_SEEK_SET))
1942 *emsg = GNUNET_strdup (strerror (errno));
1945 ret = GNUNET_DISK_file_read (fh, buf, max);
1948 *emsg = GNUNET_strdup (strerror (errno));
1956 * Task that creates the initial (top-level) download
1957 * request for the file.
1959 * @param cls the 'struct GNUNET_FS_DownloadContext'
1960 * @param tc scheduler context
1963 GNUNET_FS_download_start_task_ (void *cls,
1964 const struct GNUNET_SCHEDULER_TaskContext *tc)
1966 struct GNUNET_FS_DownloadContext *dc = cls;
1967 struct GNUNET_FS_ProgressInfo pi;
1968 struct GNUNET_DISK_FileHandle *fh;
1971 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1972 "Start task running...\n");
1974 dc->task = GNUNET_SCHEDULER_NO_TASK;
1975 if (dc->length == 0)
1977 /* no bytes required! */
1978 if (dc->filename != NULL)
1980 fh = GNUNET_DISK_file_open (dc->filename != NULL
1982 : dc->temp_filename,
1983 GNUNET_DISK_OPEN_READWRITE |
1984 GNUNET_DISK_OPEN_CREATE |
1985 ( (0 == GNUNET_FS_uri_chk_get_file_size (dc->uri))
1986 ? GNUNET_DISK_OPEN_TRUNCATE : 0),
1987 GNUNET_DISK_PERM_USER_READ |
1988 GNUNET_DISK_PERM_USER_WRITE |
1989 GNUNET_DISK_PERM_GROUP_READ |
1990 GNUNET_DISK_PERM_OTHER_READ);
1991 GNUNET_DISK_file_close (fh);
1993 GNUNET_FS_download_sync_ (dc);
1994 check_completed (dc);
1997 if (dc->emsg != NULL)
1999 if (dc->top_request == NULL)
2001 dc->top_request = create_download_request (NULL, dc->treedepth - 1, 0,
2002 dc->offset, dc->length);
2003 dc->top_request->state = BRS_CHK_SET;
2004 dc->top_request->chk = (dc->uri->type == chk)
2005 ? dc->uri->data.chk.chk
2006 : dc->uri->data.loc.fi.chk;
2008 GNUNET_FS_download_sync_ (dc);
2009 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
2010 pi.value.download.specifics.start.meta = dc->meta;
2011 GNUNET_FS_download_make_status_ (&pi, dc);
2013 GNUNET_FS_download_start_downloading_ (dc);
2014 /* attempt reconstruction from disk */
2015 if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
2016 dc->rfh = GNUNET_DISK_file_open (dc->filename,
2017 GNUNET_DISK_OPEN_READ,
2018 GNUNET_DISK_PERM_NONE);
2019 if (dc->top_request->state == BRS_CHK_SET)
2021 if (dc->rfh != NULL)
2023 /* first, try top-down */
2024 try_top_down_reconstruction (dc, dc->top_request);
2025 switch (dc->top_request->state)
2029 case BRS_DOWNLOAD_DOWN:
2030 break; /* normal, some blocks already down */
2031 case BRS_DOWNLOAD_UP:
2032 /* already done entirely, party! */
2033 dc->completed = dc->length;
2034 GNUNET_FS_download_sync_ (dc);
2035 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
2036 /* slightly ugly: no data provided to callee; maybe mmap the
2037 file instead? Or is 'data' pure convenience!? */
2038 pi.value.download.specifics.progress.data = NULL;
2039 pi.value.download.specifics.progress.offset = dc->offset;
2040 pi.value.download.specifics.progress.data_len = dc->length;
2041 pi.value.download.specifics.progress.depth = 0;
2042 GNUNET_FS_download_make_status_ (&pi, dc);
2043 if (dc->rfh != NULL)
2045 /* avoid hanging on to file handle longer than
2047 GNUNET_DISK_file_close (dc->rfh);
2050 check_completed (dc);
2053 GNUNET_asprintf (&dc->emsg,
2055 GNUNET_FS_download_sync_ (dc);
2056 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
2057 pi.value.download.specifics.error.message = dc->emsg;
2058 GNUNET_FS_download_make_status_ (&pi, dc);
2066 /* attempt reconstruction from meta data */
2067 if (GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE)
2069 GNUNET_CONTAINER_meta_data_iterate (dc->meta,
2072 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2074 if (dc->rfh != NULL)
2076 /* avoid hanging on to file handle longer than
2078 GNUNET_DISK_file_close (dc->rfh);
2081 return; /* finished, status update was already done for us */
2084 if (dc->rfh != NULL)
2086 /* finally, try bottom-up */
2087 dc->te = GNUNET_FS_tree_encoder_create (dc->h,
2094 dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
2099 /* simple, top-level download */
2100 schedule_block_download (dc,
2103 if (dc->top_request->state == BRS_DOWNLOAD_UP)
2104 check_completed (dc);
2109 * Create SUSPEND event for the given download operation
2110 * and then clean up our state (without stop signal).
2112 * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
2115 GNUNET_FS_download_signal_suspend_ (void *cls)
2117 struct GNUNET_FS_DownloadContext *dc = cls;
2118 struct GNUNET_FS_ProgressInfo pi;
2120 if (dc->top != NULL)
2121 GNUNET_FS_end_top (dc->h, dc->top);
2122 while (NULL != dc->child_head)
2123 GNUNET_FS_download_signal_suspend_ (dc->child_head);
2124 if (dc->search != NULL)
2126 dc->search->download = NULL;
2129 if (dc->job_queue != NULL)
2131 GNUNET_FS_dequeue_ (dc->job_queue);
2132 dc->job_queue = NULL;
2134 if (dc->parent != NULL)
2135 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2136 dc->parent->child_tail,
2138 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2140 GNUNET_SCHEDULER_cancel (dc->task);
2141 dc->task = GNUNET_SCHEDULER_NO_TASK;
2143 pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
2144 GNUNET_FS_download_make_status_ (&pi, dc);
2147 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2150 if (dc->rfh != NULL)
2152 GNUNET_DISK_file_close (dc->rfh);
2155 GNUNET_FS_free_download_request_ (dc->top_request);
2156 if (dc->active != NULL)
2158 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2161 GNUNET_free_non_null (dc->filename);
2162 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2163 GNUNET_FS_uri_destroy (dc->uri);
2164 GNUNET_free_non_null (dc->temp_filename);
2165 GNUNET_free_non_null (dc->serialization);
2171 * Download parts of a file. Note that this will store
2172 * the blocks at the respective offset in the given file. Also, the
2173 * download is still using the blocking of the underlying FS
2174 * encoding. As a result, the download may *write* outside of the
2175 * given boundaries (if offset and length do not match the 32k FS
2176 * block boundaries). <p>
2178 * This function should be used to focus a download towards a
2179 * particular portion of the file (optimization), not to strictly
2180 * limit the download to exactly those bytes.
2182 * @param h handle to the file sharing subsystem
2183 * @param uri the URI of the file (determines what to download); CHK or LOC URI
2184 * @param meta known metadata for the file (can be NULL)
2185 * @param filename where to store the file, maybe NULL (then no file is
2186 * created on disk and data must be grabbed from the callbacks)
2187 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2188 * can be NULL (in which case we will pick a name if needed); the temporary file
2189 * may already exist, in which case we will try to use the data that is there and
2190 * if it is not what is desired, will overwrite it
2191 * @param offset at what offset should we start the download (typically 0)
2192 * @param length how many bytes should be downloaded starting at offset
2193 * @param anonymity anonymity level to use for the download
2194 * @param options various options
2195 * @param cctx initial value for the client context for this download
2196 * @param parent parent download to associate this download with (use NULL
2197 * for top-level downloads; useful for manually-triggered recursive downloads)
2198 * @return context that can be used to control this download
2200 struct GNUNET_FS_DownloadContext *
2201 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
2202 const struct GNUNET_FS_Uri *uri,
2203 const struct GNUNET_CONTAINER_MetaData *meta,
2204 const char *filename,
2205 const char *tempname,
2209 enum GNUNET_FS_DownloadOptions options,
2211 struct GNUNET_FS_DownloadContext *parent)
2213 struct GNUNET_FS_DownloadContext *dc;
2215 GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
2216 GNUNET_FS_uri_test_loc (uri) );
2218 if ( (offset + length < offset) ||
2219 (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
2225 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2226 "Starting download `%s' of %llu bytes\n",
2228 (unsigned long long) length);
2230 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2232 dc->parent = parent;
2235 GNUNET_CONTAINER_DLL_insert (parent->child_head,
2239 dc->uri = GNUNET_FS_uri_dup (uri);
2240 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
2241 dc->client_info = cctx;
2242 dc->start_time = GNUNET_TIME_absolute_get ();
2243 if (NULL != filename)
2245 dc->filename = GNUNET_strdup (filename);
2246 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2247 GNUNET_DISK_file_size (filename,
2251 if (GNUNET_FS_uri_test_loc (dc->uri))
2252 GNUNET_assert (GNUNET_OK ==
2253 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2255 dc->offset = offset;
2256 dc->length = length;
2257 dc->anonymity = anonymity;
2258 dc->options = options;
2259 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2260 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
2261 if ( (filename == NULL) &&
2262 (is_recursive_download (dc) ) )
2264 if (tempname != NULL)
2265 dc->temp_filename = GNUNET_strdup (tempname);
2267 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2271 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2272 "Download tree has depth %u\n",
2277 dc->top = GNUNET_FS_make_top (dc->h,
2278 &GNUNET_FS_download_signal_suspend_,
2281 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2288 * Download parts of a file based on a search result. The download
2289 * will be associated with the search result (and the association
2290 * will be preserved when serializing/deserializing the state).
2291 * If the search is stopped, the download will not be aborted but
2292 * be 'promoted' to a stand-alone download.
2294 * As with the other download function, this will store
2295 * the blocks at the respective offset in the given file. Also, the
2296 * download is still using the blocking of the underlying FS
2297 * encoding. As a result, the download may *write* outside of the
2298 * given boundaries (if offset and length do not match the 32k FS
2299 * block boundaries). <p>
2301 * The given range can be used to focus a download towards a
2302 * particular portion of the file (optimization), not to strictly
2303 * limit the download to exactly those bytes.
2305 * @param h handle to the file sharing subsystem
2306 * @param sr the search result to use for the download (determines uri and
2307 * meta data and associations)
2308 * @param filename where to store the file, maybe NULL (then no file is
2309 * created on disk and data must be grabbed from the callbacks)
2310 * @param tempname where to store temporary file data, not used if filename is non-NULL;
2311 * can be NULL (in which case we will pick a name if needed); the temporary file
2312 * may already exist, in which case we will try to use the data that is there and
2313 * if it is not what is desired, will overwrite it
2314 * @param offset at what offset should we start the download (typically 0)
2315 * @param length how many bytes should be downloaded starting at offset
2316 * @param anonymity anonymity level to use for the download
2317 * @param options various download options
2318 * @param cctx initial value for the client context for this download
2319 * @return context that can be used to control this download
2321 struct GNUNET_FS_DownloadContext *
2322 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
2323 struct GNUNET_FS_SearchResult *sr,
2324 const char *filename,
2325 const char *tempname,
2329 enum GNUNET_FS_DownloadOptions options,
2332 struct GNUNET_FS_DownloadContext *dc;
2334 if ( (sr == NULL) ||
2335 (sr->download != NULL) )
2340 GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri) ||
2341 GNUNET_FS_uri_test_loc (sr->uri) );
2342 if ( (offset + length < offset) ||
2343 (offset + length > sr->uri->data.chk.file_length) )
2349 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2350 "Starting download `%s' of %llu bytes\n",
2352 (unsigned long long) length);
2354 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
2358 if (sr->probe_ctx != NULL)
2360 GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
2361 sr->probe_ctx = NULL;
2363 dc->uri = GNUNET_FS_uri_dup (sr->uri);
2364 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
2365 dc->client_info = cctx;
2366 dc->start_time = GNUNET_TIME_absolute_get ();
2367 if (NULL != filename)
2369 dc->filename = GNUNET_strdup (filename);
2370 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2371 GNUNET_DISK_file_size (filename,
2375 if (GNUNET_FS_uri_test_loc (dc->uri))
2376 GNUNET_assert (GNUNET_OK ==
2377 GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
2379 dc->offset = offset;
2380 dc->length = length;
2381 dc->anonymity = anonymity;
2382 dc->options = options;
2383 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
2384 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
2385 if ( (filename == NULL) &&
2386 (is_recursive_download (dc) ) )
2388 if (tempname != NULL)
2389 dc->temp_filename = GNUNET_strdup (tempname);
2391 dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2395 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2396 "Download tree has depth %u\n",
2399 dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
2406 * Start the downloading process (by entering the queue).
2408 * @param dc our download context
2411 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
2413 if (dc->completed == dc->length)
2415 GNUNET_assert (dc->job_queue == NULL);
2416 dc->job_queue = GNUNET_FS_queue_ (dc->h,
2417 &activate_fs_download,
2418 &deactivate_fs_download,
2420 (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
2425 * Stop a download (aborts if download is incomplete).
2427 * @param dc handle for the download
2428 * @param do_delete delete files of incomplete downloads
2431 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
2434 struct GNUNET_FS_ProgressInfo pi;
2437 if (dc->top != NULL)
2438 GNUNET_FS_end_top (dc->h, dc->top);
2441 if (dc->task != GNUNET_SCHEDULER_NO_TASK)
2443 GNUNET_SCHEDULER_cancel (dc->task);
2444 dc->task = GNUNET_SCHEDULER_NO_TASK;
2446 if (dc->search != NULL)
2448 dc->search->download = NULL;
2451 if (dc->job_queue != NULL)
2453 GNUNET_FS_dequeue_ (dc->job_queue);
2454 dc->job_queue = NULL;
2458 GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL);
2461 have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
2462 while (NULL != dc->child_head)
2463 GNUNET_FS_download_stop (dc->child_head,
2465 if (dc->parent != NULL)
2466 GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
2467 dc->parent->child_tail,
2469 if (dc->serialization != NULL)
2470 GNUNET_FS_remove_sync_file_ (dc->h,
2471 ( (dc->parent != NULL) || (dc->search != NULL) )
2472 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2473 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
2475 if ( (GNUNET_YES == have_children) &&
2476 (dc->parent == NULL) )
2477 GNUNET_FS_remove_sync_dir_ (dc->h,
2478 (dc->search != NULL)
2479 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
2480 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
2482 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
2483 GNUNET_FS_download_make_status_ (&pi, dc);
2484 GNUNET_FS_free_download_request_ (dc->top_request);
2485 dc->top_request = NULL;
2486 if (dc->active != NULL)
2488 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
2491 if (dc->filename != NULL)
2493 if ( (dc->completed != dc->length) &&
2494 (GNUNET_YES == do_delete) )
2496 if (0 != UNLINK (dc->filename))
2497 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
2501 GNUNET_free (dc->filename);
2503 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
2504 GNUNET_FS_uri_destroy (dc->uri);
2505 if (NULL != dc->temp_filename)
2507 if (0 != UNLINK (dc->temp_filename))
2508 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
2511 GNUNET_free (dc->temp_filename);
2513 GNUNET_free_non_null (dc->serialization);
2517 /* end of fs_download.c */