2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - location URI suppport (can wait, easy)
27 * - check if blocks exist already (can wait, easy)
28 * - handle recursive downloads (need directory &
29 * fs-level download-parallelism management, can wait)
30 * - check if iblocks can be computed from existing blocks (can wait, hard)
31 * - persistence (can wait)
34 #include "gnunet_constants.h"
35 #include "gnunet_fs_service.h"
39 #define DEBUG_DOWNLOAD GNUNET_YES
42 * We're storing the IBLOCKS after the
43 * DBLOCKS on disk (so that we only have
44 * to truncate the file once we're done).
46 * Given the offset of a block (with respect
47 * to the DBLOCKS) and its depth, return the
48 * offset where we would store this block
52 * @param fsize overall file size
53 * @param off offset of the block in the file
54 * @param depth depth of the block in the tree
55 * @param treedepth maximum depth of the tree
56 * @return off for DBLOCKS (depth == treedepth),
57 * otherwise an offset past the end
58 * of the file that does not overlap
59 * with the range for any other block
62 compute_disk_offset (uint64_t fsize,
65 unsigned int treedepth)
68 uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
69 uint64_t loff; /* where do IBlocks for level "i" start? */
70 unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
72 if (depth == treedepth)
74 /* first IBlocks start at the end of file, rounded up
75 to full DBLOCK_SIZE */
76 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
77 lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
78 GNUNET_assert (0 == (off % DBLOCK_SIZE));
79 ioff = (off / DBLOCK_SIZE);
80 for (i=treedepth-1;i>depth;i--)
83 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
84 GNUNET_assert (lsize > 0);
85 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
86 ioff /= CHK_PER_INODE;
88 return loff + ioff * sizeof (struct ContentHashKey);
93 * Given a file of the specified treedepth and a block at the given
94 * offset and depth, calculate the offset for the CHK at the given
97 * @param offset the offset of the first
98 * DBLOCK in the subtree of the
100 * @param depth the depth of the IBLOCK in the tree
101 * @param treedepth overall depth of the tree
102 * @param k which CHK in the IBLOCK are we
104 * @return offset if k=0, otherwise an appropriately
105 * larger value (i.e., if depth = treedepth-1,
106 * the returned value should be offset+DBLOCK_SIZE)
109 compute_dblock_offset (uint64_t offset,
111 unsigned int treedepth,
115 uint64_t lsize; /* what is the size of the sum of all DBlocks
116 that a CHK at level i corresponds to? */
118 if (depth == treedepth)
121 for (i=treedepth-1;i>depth;i--)
122 lsize *= CHK_PER_INODE;
123 return offset + i * lsize;
128 * Fill in all of the generic fields for
131 * @param pi structure to fill in
132 * @param dc overall download context
135 make_download_status (struct GNUNET_FS_ProgressInfo *pi,
136 struct GNUNET_FS_DownloadContext *dc)
138 pi->value.download.dc = dc;
139 pi->value.download.cctx
141 pi->value.download.pctx
142 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
143 pi->value.download.uri
145 pi->value.download.filename
147 pi->value.download.size
149 pi->value.download.duration
150 = GNUNET_TIME_absolute_get_duration (dc->start_time);
151 pi->value.download.completed
153 pi->value.download.anonymity
159 * Schedule the download of the specified
162 * @param dc overall download this block belongs to
163 * @param chk content-hash-key of the block
164 * @param offset offset of the block in the file
165 * (for IBlocks, the offset is the lowest
166 * offset of any DBlock in the subtree under
168 * @param depth depth of the block, 0 is the root of the tree
171 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
172 const struct ContentHashKey *chk,
176 struct DownloadRequest *sm;
180 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
181 "Scheduling download at offset %llu and depth %u for `%s'\n",
182 (unsigned long long) offset,
184 GNUNET_h2s (&chk->query));
186 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
190 if ( (dc->old_file_size > off) &&
191 (dc->handle != NULL) &&
193 GNUNET_DISK_file_seek (dc->handle,
195 GNUNET_DISK_SEEK_SET) ) )
197 // FIXME: check if block exists on disk!
198 // (read block, encode, compare with
199 // query; if matches, simply return)
201 if (depth < dc->treedepth)
203 // FIXME: try if we could
204 // reconstitute this IBLOCK
205 // from the existing blocks on disk (can wait)
206 // (read block(s), encode, compare with
207 // query; if matches, simply return)
209 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
213 sm->is_pending = GNUNET_YES;
214 sm->next = dc->pending;
216 GNUNET_CONTAINER_multihashmap_put (dc->active,
219 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
224 * We've lost our connection with the FS service.
225 * Re-establish it and re-transmit all of our
228 * @param dc download context that is having trouble
231 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
235 * Compute how many bytes of data should be stored in
236 * the specified node.
238 * @param fsize overall file size
239 * @param totaldepth depth of the entire tree
240 * @param offset offset of the node
241 * @param depth depth of the node
242 * @return number of bytes stored in this node
245 calculate_block_size (uint64_t fsize,
246 unsigned int totaldepth,
256 GNUNET_assert (offset < fsize);
257 if (depth == totaldepth)
260 if (offset + ret > fsize)
261 ret = (size_t) (fsize - offset);
266 for (i = totaldepth-1; i > depth; i--)
267 rsize *= CHK_PER_INODE;
268 epos = offset + rsize * CHK_PER_INODE;
269 GNUNET_assert (epos > offset);
272 /* round up when computing #CHKs in our IBlock */
273 chks = (epos - offset + rsize - 1) / rsize;
274 GNUNET_assert (chks <= CHK_PER_INODE);
275 return chks * sizeof (struct ContentHashKey);
280 * Process a download result.
282 * @param dc our download context
283 * @param type type of the result
284 * @param data the (encrypted) response
285 * @param size size of data
288 process_result (struct GNUNET_FS_DownloadContext *dc,
293 struct GNUNET_FS_ProgressInfo pi;
294 GNUNET_HashCode query;
295 struct DownloadRequest *sm;
296 struct GNUNET_CRYPTO_AesSessionKey skey;
297 struct GNUNET_CRYPTO_AesInitializationVector iv;
302 struct ContentHashKey *chk;
306 GNUNET_CRYPTO_hash (data, size, &query);
308 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
309 "Received result for query `%s' from `%s'-service\n",
313 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
320 if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Internal error or bogus download URI\n");
329 dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
331 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
332 make_download_status (&pi, dc);
333 pi.value.download.specifics.error.message = dc->emsg;
334 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
336 /* abort all pending requests */
337 GNUNET_CLIENT_disconnect (dc->client);
341 GNUNET_assert (GNUNET_YES ==
342 GNUNET_CONTAINER_multihashmap_remove (dc->active,
345 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
346 GNUNET_CRYPTO_aes_decrypt (data,
352 if ( (NULL != dc->handle) &&
353 ( (sm->depth == dc->treedepth) ||
354 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
356 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
362 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
363 "Saving decrypted block to disk at offset %llu\n",
364 (unsigned long long) off);
367 GNUNET_DISK_file_seek (dc->handle,
369 GNUNET_DISK_SEEK_SET) ) )
370 GNUNET_asprintf (&emsg,
371 _("Failed to seek to offset %llu in file `%s': %s\n"),
372 (unsigned long long) off,
376 GNUNET_DISK_file_write (dc->handle,
379 GNUNET_asprintf (&emsg,
380 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
382 (unsigned long long) off,
388 // FIXME: make persistent
391 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
392 make_download_status (&pi, dc);
393 pi.value.download.specifics.error.message = emsg;
394 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
397 /* abort all pending requests */
398 GNUNET_CLIENT_disconnect (dc->client);
403 if (sm->depth == dc->treedepth)
406 if (sm->offset < dc->offset)
408 /* starting offset begins in the middle of pt,
409 do not count first bytes as progress */
410 GNUNET_assert (app > (dc->offset - sm->offset));
411 app -= (dc->offset - sm->offset);
413 if (sm->offset + size > dc->offset + dc->length)
415 /* end of block is after relevant range,
416 do not count last bytes as progress */
417 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
418 app -= (sm->offset + size) - (dc->offset + dc->length);
420 dc->completed += app;
423 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
424 make_download_status (&pi, dc);
425 pi.value.download.specifics.progress.data = pt;
426 pi.value.download.specifics.progress.offset = sm->offset;
427 pi.value.download.specifics.progress.data_len = size;
428 pi.value.download.specifics.progress.depth = sm->depth;
429 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
431 GNUNET_assert (dc->completed <= dc->length);
432 if (dc->completed == dc->length)
435 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
436 "Download completed, truncating file to desired length %llu\n",
437 (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
439 /* truncate file to size (since we store IBlocks at the end) */
440 if (dc->handle != NULL)
442 GNUNET_DISK_file_close (dc->handle);
444 if (0 != truncate (dc->filename,
445 GNUNET_ntohll (dc->uri->data.chk.file_length)))
446 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
450 /* signal completion */
451 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
452 make_download_status (&pi, dc);
453 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
455 GNUNET_assert (sm->depth == dc->treedepth);
457 // FIXME: make persistent
458 if (sm->depth == dc->treedepth)
461 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
462 "Triggering downloads of children (this block was at level %u)\n",
465 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
466 chk = (struct ContentHashKey*) pt;
467 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
469 off = compute_dblock_offset (sm->offset,
473 if ( (off + DBLOCK_SIZE >= dc->offset) &&
474 (off < dc->offset + dc->length) )
475 schedule_block_download (dc,
484 * Type of a function to call when we receive a message
488 * @param msg message received, NULL on timeout or fatal error
491 receive_results (void *cls,
492 const struct GNUNET_MessageHeader * msg)
494 struct GNUNET_FS_DownloadContext *dc = cls;
495 const struct ContentMessage *cm;
498 if ( (NULL == msg) ||
499 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
500 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
505 msize = ntohs (msg->size);
506 cm = (const struct ContentMessage*) msg;
510 msize - sizeof (struct ContentMessage));
511 /* continue receiving */
512 GNUNET_CLIENT_receive (dc->client,
515 GNUNET_TIME_UNIT_FOREVER_REL);
521 * We're ready to transmit a search request to the
522 * file-sharing service. Do it. If there is
523 * more than one request pending, try to send
524 * multiple or request another transmission.
527 * @param size number of bytes available in buf
528 * @param buf where the callee should write the message
529 * @return number of bytes written to buf
532 transmit_download_request (void *cls,
536 struct GNUNET_FS_DownloadContext *dc = cls;
538 struct SearchMessage *sm;
545 GNUNET_assert (size >= sizeof (struct SearchMessage));
548 while ( (dc->pending != NULL) &&
549 (size > msize + sizeof (struct SearchMessage)) )
552 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
553 "Transmitting download request for `%s' to `%s'-service\n",
554 GNUNET_h2s (&dc->pending->chk.query),
557 memset (sm, 0, sizeof (struct SearchMessage));
558 sm->header.size = htons (sizeof (struct SearchMessage));
559 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
560 sm->anonymity_level = htonl (dc->anonymity);
561 sm->target = dc->target.hashPubKey;
562 sm->query = dc->pending->chk.query;
563 dc->pending->is_pending = GNUNET_NO;
564 dc->pending = dc->pending->next;
565 msize += sizeof (struct SearchMessage);
573 * Reconnect to the FS service and transmit
576 * @param cls our download context
580 do_reconnect (void *cls,
581 const struct GNUNET_SCHEDULER_TaskContext *tc)
583 struct GNUNET_FS_DownloadContext *dc = cls;
584 struct GNUNET_CLIENT_Connection *client;
586 dc->task = GNUNET_SCHEDULER_NO_TASK;
587 client = GNUNET_CLIENT_connect (dc->h->sched,
592 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
593 "Connecting to `%s'-service failed, will try again.\n",
599 GNUNET_CLIENT_notify_transmit_ready (client,
600 sizeof (struct SearchMessage),
601 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
602 &transmit_download_request,
604 GNUNET_CLIENT_receive (client,
607 GNUNET_TIME_UNIT_FOREVER_REL);
612 * Add entries that are not yet pending back to the pending list.
614 * @param cls our download context
616 * @param entry entry of type "struct DownloadRequest"
620 retry_entry (void *cls,
621 const GNUNET_HashCode *key,
624 struct GNUNET_FS_DownloadContext *dc = cls;
625 struct DownloadRequest *dr = entry;
627 if (! dr->is_pending)
629 dr->next = dc->pending;
630 dr->is_pending = GNUNET_YES;
638 * We've lost our connection with the FS service.
639 * Re-establish it and re-transmit all of our
642 * @param dc download context that is having trouble
645 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
648 if (NULL != dc->client)
650 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
653 GNUNET_CLIENT_disconnect (dc->client);
657 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
659 GNUNET_SCHEDULER_PRIORITY_IDLE,
660 GNUNET_SCHEDULER_NO_TASK,
661 GNUNET_TIME_UNIT_SECONDS,
668 * Download parts of a file. Note that this will store
669 * the blocks at the respective offset in the given file. Also, the
670 * download is still using the blocking of the underlying FS
671 * encoding. As a result, the download may *write* outside of the
672 * given boundaries (if offset and length do not match the 32k FS
673 * block boundaries). <p>
675 * This function should be used to focus a download towards a
676 * particular portion of the file (optimization), not to strictly
677 * limit the download to exactly those bytes.
679 * @param h handle to the file sharing subsystem
680 * @param uri the URI of the file (determines what to download); CHK or LOC URI
681 * @param meta known metadata for the file (can be NULL)
682 * @param filename where to store the file, maybe NULL (then no file is
683 * created on disk and data must be grabbed from the callbacks)
684 * @param offset at what offset should we start the download (typically 0)
685 * @param length how many bytes should be downloaded starting at offset
686 * @param anonymity anonymity level to use for the download
687 * @param options various options
688 * @param parent parent download to associate this download with (use NULL
689 * for top-level downloads; useful for manually-triggered recursive downloads)
690 * @return context that can be used to control this download
692 struct GNUNET_FS_DownloadContext *
693 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
694 const struct GNUNET_FS_Uri *uri,
695 const struct GNUNET_CONTAINER_MetaData *meta,
696 const char *filename,
700 enum GNUNET_FS_DownloadOptions options,
701 struct GNUNET_FS_DownloadContext *parent)
703 struct GNUNET_FS_ProgressInfo pi;
704 struct GNUNET_FS_DownloadContext *dc;
705 struct GNUNET_CLIENT_Connection *client;
707 client = GNUNET_CLIENT_connect (h->sched,
712 // FIXME: add support for "loc" URIs!
713 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
714 if ( (offset + length < offset) ||
715 (offset + length > uri->data.chk.file_length) )
721 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
722 "Starting download `%s' of %llu bytes\n",
724 (unsigned long long) length);
726 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
730 dc->uri = GNUNET_FS_uri_dup (uri);
731 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
732 if (NULL != filename)
734 dc->filename = GNUNET_strdup (filename);
735 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
736 GNUNET_DISK_file_size (filename,
739 dc->handle = GNUNET_DISK_file_open (filename,
740 GNUNET_DISK_OPEN_READWRITE |
741 GNUNET_DISK_OPEN_CREATE,
742 GNUNET_DISK_PERM_USER_READ |
743 GNUNET_DISK_PERM_USER_WRITE |
744 GNUNET_DISK_PERM_GROUP_READ |
745 GNUNET_DISK_PERM_OTHER_READ);
746 if (dc->handle == NULL)
748 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
749 _("Download failed: could not open file `%s': %s\n"),
752 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
753 GNUNET_FS_uri_destroy (dc->uri);
754 GNUNET_free (dc->filename);
755 GNUNET_CLIENT_disconnect (dc->client);
760 // FIXME: set "dc->target" for LOC uris!
763 dc->anonymity = anonymity;
764 dc->options = options;
765 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
766 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
768 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
769 "Download tree has depth %u\n",
772 // FIXME: make persistent
773 schedule_block_download (dc,
774 &dc->uri->data.chk.chk,
777 GNUNET_CLIENT_notify_transmit_ready (client,
778 sizeof (struct SearchMessage),
779 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
780 &transmit_download_request,
782 GNUNET_CLIENT_receive (client,
785 GNUNET_TIME_UNIT_FOREVER_REL);
786 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
787 make_download_status (&pi, dc);
788 pi.value.download.specifics.start.meta = meta;
789 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
797 * Free entries in the map.
799 * @param cls unused (NULL)
801 * @param entry entry of type "struct DownloadRequest" which is freed
805 free_entry (void *cls,
806 const GNUNET_HashCode *key,
815 * Stop a download (aborts if download is incomplete).
817 * @param dc handle for the download
818 * @param do_delete delete files of incomplete downloads
821 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
824 struct GNUNET_FS_ProgressInfo pi;
826 // FIXME: make unpersistent
827 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
828 make_download_status (&pi, dc);
829 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
832 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
833 GNUNET_SCHEDULER_cancel (dc->h->sched,
835 if (NULL != dc->client)
836 GNUNET_CLIENT_disconnect (dc->client);
837 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
840 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
841 if (dc->filename != NULL)
843 if (NULL != dc->handle)
844 GNUNET_DISK_file_close (dc->handle);
845 if ( (dc->completed != dc->length) &&
846 (GNUNET_YES == do_delete) )
848 if (0 != UNLINK (dc->filename))
849 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
853 GNUNET_free (dc->filename);
855 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
856 GNUNET_FS_uri_destroy (dc->uri);
866 * Check if self block is already present on the drive. If the block
867 * is a dblock and present, the ProgressModel is notified. If the
868 * block is present and it is an iblock, downloading the children is
871 * Also checks if the block is within the range of blocks
872 * that we are supposed to download. If not, the method
873 * returns as if the block is present but does NOT signal
876 * @param node that is checked for presence
877 * @return GNUNET_YES if present, GNUNET_NO if not.
880 check_node_present (const struct Node *node)
888 size = get_node_size (node);
889 /* first check if node is within range.
890 For now, keeping it simple, we only do
891 this for level-0 nodes */
892 if ((node->level == 0) &&
893 ((node->offset + size < node->ctx->offset) ||
894 (node->offset >= node->ctx->offset + node->ctx->length)))
896 data = GNUNET_malloc (size);
898 res = read_from_files (node->ctx, node->level, node->offset, data, size);
901 GNUNET_hash (data, size, &hc);
902 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
904 notify_client_about_progress (node, data, size);
906 iblock_download_children (node, data, size);
917 /* end of fs_download.c */