2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - location URI suppport (can wait, easy)
27 * - check if blocks exist already (can wait, easy)
28 * - handle recursive downloads (need directory &
29 * fs-level download-parallelism management, can wait)
30 * - check if iblocks can be computed from existing blocks (can wait, hard)
31 * - persistence (can wait)
34 #include "gnunet_constants.h"
35 #include "gnunet_fs_service.h"
39 #define DEBUG_DOWNLOAD GNUNET_YES
42 * We're storing the IBLOCKS after the
43 * DBLOCKS on disk (so that we only have
44 * to truncate the file once we're done).
46 * Given the offset of a block (with respect
47 * to the DBLOCKS) and its depth, return the
48 * offset where we would store this block
52 * @param fsize overall file size
53 * @param off offset of the block in the file
54 * @param depth depth of the block in the tree
55 * @param treedepth maximum depth of the tree
56 * @return off for DBLOCKS (depth == treedepth),
57 * otherwise an offset past the end
58 * of the file that does not overlap
59 * with the range for any other block
62 compute_disk_offset (uint64_t fsize,
65 unsigned int treedepth)
68 uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
69 uint64_t loff; /* where do IBlocks for level "i" start? */
70 unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
72 if (depth == treedepth)
74 /* first IBlocks start at the end of file, rounded up
75 to full DBLOCK_SIZE */
76 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
77 lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
78 GNUNET_assert (0 == (off % DBLOCK_SIZE));
79 ioff = (off / DBLOCK_SIZE);
80 for (i=treedepth-1;i>depth;i--)
83 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
84 GNUNET_assert (lsize > 0);
85 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
86 ioff /= CHK_PER_INODE;
88 return loff + ioff * sizeof (struct ContentHashKey);
93 * Given a file of the specified treedepth and a block at the given
94 * offset and depth, calculate the offset for the CHK at the given
97 * @param offset the offset of the first
98 * DBLOCK in the subtree of the
100 * @param depth the depth of the IBLOCK in the tree
101 * @param treedepth overall depth of the tree
102 * @param k which CHK in the IBLOCK are we
104 * @return offset if k=0, otherwise an appropriately
105 * larger value (i.e., if depth = treedepth-1,
106 * the returned value should be offset+DBLOCK_SIZE)
109 compute_dblock_offset (uint64_t offset,
111 unsigned int treedepth,
115 uint64_t lsize; /* what is the size of the sum of all DBlocks
116 that a CHK at level i corresponds to? */
118 if (depth == treedepth)
121 for (i=treedepth-1;i>depth;i--)
122 lsize *= CHK_PER_INODE;
123 return offset + i * lsize;
128 * Fill in all of the generic fields for
131 * @param pc structure to fill in
132 * @param dc overall download context
135 make_download_status (struct GNUNET_FS_ProgressInfo *pi,
136 struct GNUNET_FS_DownloadContext *dc)
138 pi->value.download.dc = dc;
139 pi->value.download.cctx
141 pi->value.download.pctx
142 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
143 pi->value.download.uri
145 pi->value.download.filename
147 pi->value.download.length
149 pi->value.download.duration
150 = GNUNET_TIME_absolute_get_duration (dc->start_time);
151 pi->value.download.completed
153 pi->value.download.anonymity
159 * Schedule the download of the specified
162 * @param dc overall download this block belongs to
163 * @param chk content-hash-key of the block
164 * @param offset offset of the block in the file
165 * (for IBlocks, the offset is the lowest
166 * offset of any DBlock in the subtree under
168 * @param depth depth of the block, 0 is the root of the tree
171 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
172 const struct ContentHashKey *chk,
176 struct DownloadRequest *sm;
179 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
183 if ( (dc->old_file_size > off) &&
184 (dc->handle != NULL) &&
186 GNUNET_DISK_file_seek (dc->handle,
188 GNUNET_DISK_SEEK_SET) ) )
190 // FIXME: check if block exists on disk!
191 // (read block, encode, compare with
192 // query; if matches, simply return)
194 if (depth < dc->treedepth)
196 // FIXME: try if we could
197 // reconstitute this IBLOCK
198 // from the existing blocks on disk (can wait)
199 // (read block(s), encode, compare with
200 // query; if matches, simply return)
202 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
206 sm->is_pending = GNUNET_YES;
207 sm->next = dc->pending;
209 GNUNET_CONTAINER_multihashmap_put (dc->active,
212 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
217 * We've lost our connection with the FS service.
218 * Re-establish it and re-transmit all of our
221 * @param dc download context that is having trouble
224 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
228 * Compute how many bytes of data should be stored in
229 * the specified node.
231 * @param fsize overall file size
232 * @param off offset of the node
233 * @param depth depth of the node
234 * @return number of bytes stored in this node
237 calculate_block_size (uint64_t fsize,
238 unsigned int totaldepth,
248 GNUNET_assert (offset < fsize);
249 if (depth == totaldepth)
252 if (offset + ret > fsize)
253 ret = (size_t) (fsize - offset);
258 for (i = totaldepth-1; i > depth; i--)
259 rsize *= CHK_PER_INODE;
260 epos = offset + rsize * CHK_PER_INODE;
261 GNUNET_assert (epos > offset);
264 /* round up when computing #CHKs in our IBlock */
265 chks = (epos - offset + rsize - 1) / rsize;
266 GNUNET_assert (chks <= CHK_PER_INODE);
267 return chks * sizeof (struct ContentHashKey);
272 * Process a search result.
274 * @param sc our search context
275 * @param type type of the result
276 * @param data the (encrypted) response
277 * @param size size of data
280 process_result (struct GNUNET_FS_DownloadContext *dc,
285 struct GNUNET_FS_ProgressInfo pi;
286 GNUNET_HashCode query;
287 struct DownloadRequest *sm;
288 struct GNUNET_CRYPTO_AesSessionKey skey;
289 struct GNUNET_CRYPTO_AesInitializationVector iv;
294 struct ContentHashKey *chk;
297 GNUNET_CRYPTO_hash (data, size, &query);
298 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
305 if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
310 dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
312 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
313 make_download_status (&pi, dc);
314 pi.value.download.specifics.error.message = dc->emsg;
315 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
317 /* abort all pending requests */
318 GNUNET_CLIENT_disconnect (dc->client);
322 GNUNET_assert (GNUNET_YES ==
323 GNUNET_CONTAINER_multihashmap_remove (dc->active,
326 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
327 GNUNET_CRYPTO_aes_decrypt (data,
333 if ( (NULL != dc->handle) &&
334 ( (sm->depth == dc->treedepth) ||
335 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
337 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
343 GNUNET_DISK_file_seek (dc->handle,
345 GNUNET_DISK_SEEK_SET) ) )
346 GNUNET_asprintf (&emsg,
347 _("Failed to seek to offset %llu in file `%s': %s\n"),
348 (unsigned long long) off,
352 GNUNET_DISK_file_write (dc->handle,
355 GNUNET_asprintf (&emsg,
356 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
358 (unsigned long long) off,
364 // FIXME: make persistent
367 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
368 make_download_status (&pi, dc);
369 pi.value.download.specifics.error.message = emsg;
370 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
373 /* abort all pending requests */
374 GNUNET_CLIENT_disconnect (dc->client);
379 if (sm->depth == dc->treedepth)
382 if (sm->offset < dc->offset)
384 /* starting offset begins in the middle of pt,
385 do not count first bytes as progress */
386 GNUNET_assert (app > (dc->offset - sm->offset));
387 app -= (dc->offset - sm->offset);
389 if (sm->offset + size > dc->offset + dc->length)
391 /* end of block is after relevant range,
392 do not count last bytes as progress */
393 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
394 app -= (sm->offset + size) - (dc->offset + dc->length);
396 dc->completed += app;
399 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
400 make_download_status (&pi, dc);
401 pi.value.download.specifics.progress.data = pt;
402 pi.value.download.specifics.progress.offset = sm->offset;
403 pi.value.download.specifics.progress.data_len = size;
404 pi.value.download.specifics.progress.depth = sm->depth;
405 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
407 GNUNET_assert (dc->completed <= dc->length);
408 if (dc->completed == dc->length)
410 /* truncate file to size (since we store IBlocks at the end) */
411 if (dc->handle != NULL)
413 GNUNET_DISK_file_close (dc->handle);
415 if (0 != truncate (dc->filename,
416 GNUNET_ntohll (dc->uri->data.chk.file_length)))
417 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
421 /* signal completion */
422 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
423 make_download_status (&pi, dc);
424 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
426 GNUNET_assert (sm->depth == dc->treedepth);
428 // FIXME: make persistent
429 if (sm->depth == dc->treedepth)
431 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
432 chk = (struct ContentHashKey*) pt;
433 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
435 off = compute_dblock_offset (sm->offset,
439 if ( (off + DBLOCK_SIZE >= dc->offset) &&
440 (off < dc->offset + dc->length) )
441 schedule_block_download (dc,
450 * Type of a function to call when we receive a message
454 * @param msg message received, NULL on timeout or fatal error
457 receive_results (void *cls,
458 const struct GNUNET_MessageHeader * msg)
460 struct GNUNET_FS_DownloadContext *dc = cls;
461 const struct ContentMessage *cm;
464 if ( (NULL == msg) ||
465 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
466 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
471 msize = ntohs (msg->size);
472 cm = (const struct ContentMessage*) msg;
476 msize - sizeof (struct ContentMessage));
477 /* continue receiving */
478 GNUNET_CLIENT_receive (dc->client,
481 GNUNET_TIME_UNIT_FOREVER_REL);
487 * We're ready to transmit a search request to the
488 * file-sharing service. Do it. If there is
489 * more than one request pending, try to send
490 * multiple or request another transmission.
493 * @param size number of bytes available in buf
494 * @param buf where the callee should write the message
495 * @return number of bytes written to buf
498 transmit_download_request (void *cls,
502 struct GNUNET_FS_DownloadContext *dc = cls;
504 struct SearchMessage *sm;
511 GNUNET_assert (size >= sizeof (struct SearchMessage));
514 while ( (dc->pending == NULL) &&
515 (size > msize + sizeof (struct SearchMessage)) )
517 memset (sm, 0, sizeof (struct SearchMessage));
518 sm->header.size = htons (sizeof (struct SearchMessage));
519 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
520 sm->anonymity_level = htonl (dc->anonymity);
521 sm->target = dc->target.hashPubKey;
522 sm->query = dc->pending->chk.query;
523 dc->pending->is_pending = GNUNET_NO;
524 dc->pending = dc->pending->next;
525 msize += sizeof (struct SearchMessage);
533 * Reconnect to the FS service and transmit
536 * @param cls our download context
540 do_reconnect (void *cls,
541 const struct GNUNET_SCHEDULER_TaskContext *tc)
543 struct GNUNET_FS_DownloadContext *dc = cls;
544 struct GNUNET_CLIENT_Connection *client;
546 dc->task = GNUNET_SCHEDULER_NO_TASK;
547 client = GNUNET_CLIENT_connect (dc->h->sched,
556 GNUNET_CLIENT_notify_transmit_ready (client,
557 sizeof (struct SearchMessage),
558 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
559 &transmit_download_request,
561 GNUNET_CLIENT_receive (client,
564 GNUNET_TIME_UNIT_FOREVER_REL);
569 * Add entries that are not yet pending back to
572 * @param cls our download context
574 * @param entry entry of type "struct DownloadRequest"
578 retry_entry (void *cls,
579 const GNUNET_HashCode *key,
582 struct GNUNET_FS_DownloadContext *dc = cls;
583 struct DownloadRequest *dr = entry;
585 if (! dr->is_pending)
587 dr->next = dc->pending;
588 dr->is_pending = GNUNET_YES;
596 * We've lost our connection with the FS service.
597 * Re-establish it and re-transmit all of our
600 * @param dc download context that is having trouble
603 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
606 if (NULL != dc->client)
608 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
611 GNUNET_CLIENT_disconnect (dc->client);
615 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
617 GNUNET_SCHEDULER_PRIORITY_IDLE,
618 GNUNET_SCHEDULER_NO_TASK,
619 GNUNET_TIME_UNIT_SECONDS,
626 * Download parts of a file. Note that this will store
627 * the blocks at the respective offset in the given file. Also, the
628 * download is still using the blocking of the underlying FS
629 * encoding. As a result, the download may *write* outside of the
630 * given boundaries (if offset and length do not match the 32k FS
631 * block boundaries). <p>
633 * This function should be used to focus a download towards a
634 * particular portion of the file (optimization), not to strictly
635 * limit the download to exactly those bytes.
637 * @param h handle to the file sharing subsystem
638 * @param uri the URI of the file (determines what to download); CHK or LOC URI
639 * @param meta known metadata for the file (can be NULL)
640 * @param filename where to store the file, maybe NULL (then no file is
641 * created on disk and data must be grabbed from the callbacks)
642 * @param offset at what offset should we start the download (typically 0)
643 * @param length how many bytes should be downloaded starting at offset
644 * @param anonymity anonymity level to use for the download
645 * @param options various options
646 * @param parent parent download to associate this download with (use NULL
647 * for top-level downloads; useful for manually-triggered recursive downloads)
648 * @return context that can be used to control this download
650 struct GNUNET_FS_DownloadContext *
651 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
652 const struct GNUNET_FS_Uri *uri,
653 const struct GNUNET_CONTAINER_MetaData *meta,
654 const char *filename,
658 enum GNUNET_FS_DownloadOptions options,
659 struct GNUNET_FS_DownloadContext *parent)
661 struct GNUNET_FS_ProgressInfo pi;
662 struct GNUNET_FS_DownloadContext *dc;
663 struct GNUNET_CLIENT_Connection *client;
665 client = GNUNET_CLIENT_connect (h->sched,
670 // FIXME: add support for "loc" URIs!
671 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
672 if ( (offset + length < offset) ||
673 (offset + length > uri->data.chk.file_length) )
678 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
682 dc->uri = GNUNET_FS_uri_dup (uri);
683 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
684 if (NULL != filename)
686 dc->filename = GNUNET_strdup (filename);
687 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
688 GNUNET_DISK_file_size (filename,
691 dc->handle = GNUNET_DISK_file_open (filename,
692 GNUNET_DISK_OPEN_READWRITE |
693 GNUNET_DISK_OPEN_CREATE,
694 GNUNET_DISK_PERM_USER_READ |
695 GNUNET_DISK_PERM_USER_WRITE |
696 GNUNET_DISK_PERM_GROUP_READ |
697 GNUNET_DISK_PERM_OTHER_READ);
698 if (dc->handle == NULL)
700 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
701 _("Download failed: could not open file `%s': %s\n"),
704 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
705 GNUNET_FS_uri_destroy (dc->uri);
706 GNUNET_free (dc->filename);
707 GNUNET_CLIENT_disconnect (dc->client);
712 // FIXME: set "dc->target" for LOC uris!
715 dc->anonymity = anonymity;
716 dc->options = options;
717 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
718 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
719 // FIXME: make persistent
720 schedule_block_download (dc,
721 &dc->uri->data.chk.chk,
724 GNUNET_CLIENT_notify_transmit_ready (client,
725 sizeof (struct SearchMessage),
726 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
727 &transmit_download_request,
729 GNUNET_CLIENT_receive (client,
732 GNUNET_TIME_UNIT_FOREVER_REL);
733 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
734 make_download_status (&pi, dc);
735 pi.value.download.specifics.start.meta = meta;
736 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
744 * Free entries in the map.
746 * @param cls unused (NULL)
748 * @param entry entry of type "struct DownloadRequest" which is freed
752 free_entry (void *cls,
753 const GNUNET_HashCode *key,
762 * Stop a download (aborts if download is incomplete).
764 * @param dc handle for the download
765 * @param do_delete delete files of incomplete downloads
768 GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
771 struct GNUNET_FS_ProgressInfo pi;
773 // FIXME: make unpersistent
774 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
775 make_download_status (&pi, dc);
776 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
779 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
780 GNUNET_SCHEDULER_cancel (dc->h->sched,
782 if (NULL != dc->client)
783 GNUNET_CLIENT_disconnect (dc->client);
784 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
787 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
788 if (dc->filename != NULL)
790 if (NULL != dc->handle)
791 GNUNET_DISK_file_close (dc->handle);
792 if ( (dc->completed != dc->length) &&
793 (GNUNET_YES == do_delete) )
795 if (0 != UNLINK (dc->filename))
796 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
800 GNUNET_free (dc->filename);
802 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
803 GNUNET_FS_uri_destroy (dc->uri);
813 * Check if self block is already present on the drive. If the block
814 * is a dblock and present, the ProgressModel is notified. If the
815 * block is present and it is an iblock, downloading the children is
818 * Also checks if the block is within the range of blocks
819 * that we are supposed to download. If not, the method
820 * returns as if the block is present but does NOT signal
823 * @param node that is checked for presence
824 * @return GNUNET_YES if present, GNUNET_NO if not.
827 check_node_present (const struct Node *node)
835 size = get_node_size (node);
836 /* first check if node is within range.
837 For now, keeping it simple, we only do
838 this for level-0 nodes */
839 if ((node->level == 0) &&
840 ((node->offset + size < node->ctx->offset) ||
841 (node->offset >= node->ctx->offset + node->ctx->length)))
843 data = GNUNET_malloc (size);
845 res = read_from_files (node->ctx, node->level, node->offset, data, size);
848 GNUNET_hash (data, size, &hc);
849 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
851 notify_client_about_progress (node, data, size);
853 iblock_download_children (node, data, size);
864 /* end of fs_download.c */