2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - location URI suppport (can wait, easy)
27 * - check if blocks exist already (can wait, easy)
28 * - handle recursive downloads (need directory &
29 * fs-level download-parallelism management, can wait)
30 * - check if iblocks can be computed from existing blocks (can wait, hard)
31 * - persistence (can wait)
34 #include "gnunet_constants.h"
35 #include "gnunet_fs_service.h"
39 #define DEBUG_DOWNLOAD GNUNET_YES
42 * We're storing the IBLOCKS after the
43 * DBLOCKS on disk (so that we only have
44 * to truncate the file once we're done).
46 * Given the offset of a block (with respect
47 * to the DBLOCKS) and its depth, return the
48 * offset where we would store this block
52 * @param fsize overall file size
53 * @param off offset of the block in the file
54 * @param depth depth of the block in the tree
55 * @param treedepth maximum depth of the tree
56 * @return off for DBLOCKS (depth == treedepth),
57 * otherwise an offset past the end
58 * of the file that does not overlap
59 * with the range for any other block
62 compute_disk_offset (uint64_t fsize,
65 unsigned int treedepth)
68 uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
69 uint64_t loff; /* where do IBlocks for level "i" start? */
70 unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
72 if (depth == treedepth)
74 /* first IBlocks start at the end of file, rounded up
75 to full DBLOCK_SIZE */
76 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
77 lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
78 GNUNET_assert (0 == (off % DBLOCK_SIZE));
79 ioff = (off / DBLOCK_SIZE);
80 for (i=treedepth-1;i>depth;i--)
83 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
84 GNUNET_assert (lsize > 0);
85 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
86 ioff /= CHK_PER_INODE;
88 return loff + ioff * sizeof (struct ContentHashKey);
93 * Given a file of the specified treedepth and a block at the given
94 * offset and depth, calculate the offset for the CHK at the given
97 * @param offset the offset of the first
98 * DBLOCK in the subtree of the
100 * @param depth the depth of the IBLOCK in the tree
101 * @param treedepth overall depth of the tree
102 * @param k which CHK in the IBLOCK are we
104 * @return offset if k=0, otherwise an appropriately
105 * larger value (i.e., if depth = treedepth-1,
106 * the returned value should be offset+DBLOCK_SIZE)
109 compute_dblock_offset (uint64_t offset,
111 unsigned int treedepth,
115 uint64_t lsize; /* what is the size of the sum of all DBlocks
116 that a CHK at level i corresponds to? */
118 if (depth == treedepth)
121 for (i=treedepth-1;i>depth;i--)
122 lsize *= CHK_PER_INODE;
123 return offset + i * lsize;
128 * Fill in all of the generic fields for
131 * @param pc structure to fill in
132 * @param dc overall download context
135 make_download_status (struct GNUNET_FS_ProgressInfo *pi,
136 struct GNUNET_FS_DownloadContext *dc)
138 pi->value.download.dc = dc;
139 pi->value.download.cctx
141 pi->value.download.pctx
142 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
143 pi->value.download.uri
145 pi->value.download.length
147 pi->value.download.duration
148 = GNUNET_TIME_absolute_get_duration (dc->start_time);
149 pi->value.download.completed
151 pi->value.download.anonymity
157 * Schedule the download of the specified
160 * @param dc overall download this block belongs to
161 * @param chk content-hash-key of the block
162 * @param offset offset of the block in the file
163 * (for IBlocks, the offset is the lowest
164 * offset of any DBlock in the subtree under
166 * @param depth depth of the block, 0 is the root of the tree
169 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
170 const struct ContentHashKey *chk,
174 struct DownloadRequest *sm;
177 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
181 if ( (dc->old_file_size > off) &&
182 (dc->handle != NULL) &&
184 GNUNET_DISK_file_seek (dc->handle,
186 GNUNET_DISK_SEEK_SET) ) )
188 // FIXME: check if block exists on disk!
189 // (read block, encode, compare with
190 // query; if matches, simply return)
192 if (depth < dc->treedepth)
194 // FIXME: try if we could
195 // reconstitute this IBLOCK
196 // from the existing blocks on disk (can wait)
197 // (read block(s), encode, compare with
198 // query; if matches, simply return)
200 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
204 sm->is_pending = GNUNET_YES;
205 sm->next = dc->pending;
207 GNUNET_CONTAINER_multihashmap_put (dc->active,
210 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
215 * We've lost our connection with the FS service.
216 * Re-establish it and re-transmit all of our
219 * @param dc download context that is having trouble
222 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
226 * Compute how many bytes of data should be stored in
227 * the specified node.
229 * @param fsize overall file size
230 * @param off offset of the node
231 * @param depth depth of the node
232 * @return number of bytes stored in this node
235 calculate_block_size (uint64_t fsize,
236 unsigned int totaldepth,
246 GNUNET_assert (offset < fsize);
247 if (depth == totaldepth)
250 if (offset + ret > fsize)
251 ret = (size_t) (fsize - offset);
256 for (i = totaldepth-1; i > depth; i--)
257 rsize *= CHK_PER_INODE;
258 epos = offset + rsize * CHK_PER_INODE;
259 GNUNET_assert (epos > offset);
262 /* round up when computing #CHKs in our IBlock */
263 chks = (epos - offset + rsize - 1) / rsize;
264 GNUNET_assert (chks <= CHK_PER_INODE);
265 return chks * sizeof (struct ContentHashKey);
270 * Process a search result.
272 * @param sc our search context
273 * @param type type of the result
274 * @param data the (encrypted) response
275 * @param size size of data
278 process_result (struct GNUNET_FS_DownloadContext *dc,
283 struct GNUNET_FS_ProgressInfo pi;
284 GNUNET_HashCode query;
285 struct DownloadRequest *sm;
286 struct GNUNET_CRYPTO_AesSessionKey skey;
287 struct GNUNET_CRYPTO_AesInitializationVector iv;
292 struct ContentHashKey *chk;
295 GNUNET_CRYPTO_hash (data, size, &query);
296 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
303 if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
308 dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
310 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
311 make_download_status (&pi, dc);
312 pi.value.download.specifics.error.message = dc->emsg;
313 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
315 /* abort all pending requests */
316 GNUNET_CLIENT_disconnect (dc->client);
320 GNUNET_assert (GNUNET_YES ==
321 GNUNET_CONTAINER_multihashmap_remove (dc->active,
324 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
325 GNUNET_CRYPTO_aes_decrypt (data,
331 if ( (NULL != dc->handle) &&
332 ( (sm->depth == dc->treedepth) ||
333 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
335 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
341 GNUNET_DISK_file_seek (dc->handle,
343 GNUNET_DISK_SEEK_SET) ) )
344 GNUNET_asprintf (&emsg,
345 _("Failed to seek to offset %llu in file `%s': %s\n"),
346 (unsigned long long) off,
350 GNUNET_DISK_file_write (dc->handle,
353 GNUNET_asprintf (&emsg,
354 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
356 (unsigned long long) off,
362 // FIXME: make persistent
365 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
366 make_download_status (&pi, dc);
367 pi.value.download.specifics.error.message = emsg;
368 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
371 /* abort all pending requests */
372 GNUNET_CLIENT_disconnect (dc->client);
377 if (sm->depth == dc->treedepth)
380 if (sm->offset < dc->offset)
382 /* starting offset begins in the middle of pt,
383 do not count first bytes as progress */
384 GNUNET_assert (app > (dc->offset - sm->offset));
385 app -= (dc->offset - sm->offset);
387 if (sm->offset + size > dc->offset + dc->length)
389 /* end of block is after relevant range,
390 do not count last bytes as progress */
391 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
392 app -= (sm->offset + size) - (dc->offset + dc->length);
394 dc->completed += app;
397 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
398 make_download_status (&pi, dc);
399 pi.value.download.specifics.progress.data = pt;
400 pi.value.download.specifics.progress.offset = sm->offset;
401 pi.value.download.specifics.progress.data_len = size;
402 pi.value.download.specifics.progress.depth = sm->depth;
403 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
405 GNUNET_assert (dc->completed <= dc->length);
406 if (dc->completed == dc->length)
408 /* truncate file to size (since we store IBlocks at the end) */
409 if (dc->handle != NULL)
411 GNUNET_DISK_file_close (dc->handle);
413 if (0 != truncate (dc->filename,
414 GNUNET_ntohll (dc->uri->data.chk.file_length)))
415 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
419 /* signal completion */
420 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
421 make_download_status (&pi, dc);
422 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
424 GNUNET_assert (sm->depth == dc->treedepth);
426 // FIXME: make persistent
427 if (sm->depth == dc->treedepth)
429 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
430 chk = (struct ContentHashKey*) pt;
431 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
433 off = compute_dblock_offset (sm->offset,
437 if ( (off + DBLOCK_SIZE >= dc->offset) &&
438 (off < dc->offset + dc->length) )
439 schedule_block_download (dc,
448 * Type of a function to call when we receive a message
452 * @param msg message received, NULL on timeout or fatal error
455 receive_results (void *cls,
456 const struct GNUNET_MessageHeader * msg)
458 struct GNUNET_FS_DownloadContext *dc = cls;
459 const struct ContentMessage *cm;
462 if ( (NULL == msg) ||
463 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
464 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
469 msize = ntohs (msg->size);
470 cm = (const struct ContentMessage*) msg;
474 msize - sizeof (struct ContentMessage));
475 /* continue receiving */
476 GNUNET_CLIENT_receive (dc->client,
479 GNUNET_TIME_UNIT_FOREVER_REL);
485 * We're ready to transmit a search request to the
486 * file-sharing service. Do it. If there is
487 * more than one request pending, try to send
488 * multiple or request another transmission.
491 * @param size number of bytes available in buf
492 * @param buf where the callee should write the message
493 * @return number of bytes written to buf
496 transmit_download_request (void *cls,
500 struct GNUNET_FS_DownloadContext *dc = cls;
502 struct SearchMessage *sm;
509 GNUNET_assert (size >= sizeof (struct SearchMessage));
512 while ( (dc->pending == NULL) &&
513 (size > msize + sizeof (struct SearchMessage)) )
515 memset (sm, 0, sizeof (struct SearchMessage));
516 sm->header.size = htons (sizeof (struct SearchMessage));
517 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
518 sm->anonymity_level = htonl (dc->anonymity);
519 sm->target = dc->target.hashPubKey;
520 sm->query = dc->pending->chk.query;
521 dc->pending->is_pending = GNUNET_NO;
522 dc->pending = dc->pending->next;
523 msize += sizeof (struct SearchMessage);
531 * Reconnect to the FS service and transmit
534 * @param cls our download context
538 do_reconnect (void *cls,
539 const struct GNUNET_SCHEDULER_TaskContext *tc)
541 struct GNUNET_FS_DownloadContext *dc = cls;
542 struct GNUNET_CLIENT_Connection *client;
544 dc->task = GNUNET_SCHEDULER_NO_TASK;
545 client = GNUNET_CLIENT_connect (dc->h->sched,
554 GNUNET_CLIENT_notify_transmit_ready (client,
555 sizeof (struct SearchMessage),
556 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
557 &transmit_download_request,
559 GNUNET_CLIENT_receive (client,
562 GNUNET_TIME_UNIT_FOREVER_REL);
567 * Add entries that are not yet pending back to
570 * @param cls our download context
572 * @param entry entry of type "struct DownloadRequest"
576 retry_entry (void *cls,
577 const GNUNET_HashCode *key,
580 struct GNUNET_FS_DownloadContext *dc = cls;
581 struct DownloadRequest *dr = entry;
583 if (! dr->is_pending)
585 dr->next = dc->pending;
586 dr->is_pending = GNUNET_YES;
594 * We've lost our connection with the FS service.
595 * Re-establish it and re-transmit all of our
598 * @param dc download context that is having trouble
601 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
604 if (NULL != dc->client)
606 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
609 GNUNET_CLIENT_disconnect (dc->client);
613 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
615 GNUNET_SCHEDULER_PRIORITY_IDLE,
616 GNUNET_SCHEDULER_NO_TASK,
617 GNUNET_TIME_UNIT_SECONDS,
624 * Download parts of a file. Note that this will store
625 * the blocks at the respective offset in the given file. Also, the
626 * download is still using the blocking of the underlying FS
627 * encoding. As a result, the download may *write* outside of the
628 * given boundaries (if offset and length do not match the 32k FS
629 * block boundaries). <p>
631 * This function should be used to focus a download towards a
632 * particular portion of the file (optimization), not to strictly
633 * limit the download to exactly those bytes.
635 * @param h handle to the file sharing subsystem
636 * @param uri the URI of the file (determines what to download); CHK or LOC URI
637 * @param meta known metadata for the file (can be NULL)
638 * @param filename where to store the file, maybe NULL (then no file is
639 * created on disk and data must be grabbed from the callbacks)
640 * @param offset at what offset should we start the download (typically 0)
641 * @param length how many bytes should be downloaded starting at offset
642 * @param anonymity anonymity level to use for the download
643 * @param options various options
644 * @param parent parent download to associate this download with (use NULL
645 * for top-level downloads; useful for manually-triggered recursive downloads)
646 * @return context that can be used to control this download
648 struct GNUNET_FS_DownloadContext *
649 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
650 const struct GNUNET_FS_Uri *uri,
651 const struct GNUNET_CONTAINER_MetaData *meta,
652 const char *filename,
656 enum GNUNET_FS_DownloadOptions options,
657 struct GNUNET_FS_DownloadContext *parent)
659 struct GNUNET_FS_ProgressInfo pi;
660 struct GNUNET_FS_DownloadContext *dc;
661 struct GNUNET_CLIENT_Connection *client;
663 client = GNUNET_CLIENT_connect (h->sched,
668 // FIXME: add support for "loc" URIs!
669 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
670 if ( (dc->offset + dc->length < dc->offset) ||
671 (dc->offset + dc->length > uri->data.chk.file_length) )
676 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
680 dc->uri = GNUNET_FS_uri_dup (uri);
681 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
682 if (NULL != filename)
684 dc->filename = GNUNET_strdup (filename);
685 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
686 GNUNET_DISK_file_size (filename,
689 dc->handle = GNUNET_DISK_file_open (filename,
690 GNUNET_DISK_OPEN_READWRITE |
691 GNUNET_DISK_OPEN_CREATE,
692 GNUNET_DISK_PERM_USER_READ |
693 GNUNET_DISK_PERM_USER_WRITE |
694 GNUNET_DISK_PERM_GROUP_READ |
695 GNUNET_DISK_PERM_OTHER_READ);
696 if (dc->handle == NULL)
698 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
699 _("Download failed: could not open file `%s': %s\n"),
702 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
703 GNUNET_FS_uri_destroy (dc->uri);
704 GNUNET_free (dc->filename);
705 GNUNET_CLIENT_disconnect (dc->client);
710 // FIXME: set "dc->target" for LOC uris!
713 dc->anonymity = anonymity;
714 dc->options = options;
715 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
716 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
717 // FIXME: make persistent
718 schedule_block_download (dc,
719 &dc->uri->data.chk.chk,
722 GNUNET_CLIENT_notify_transmit_ready (client,
723 sizeof (struct SearchMessage),
724 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
725 &transmit_download_request,
727 GNUNET_CLIENT_receive (client,
730 GNUNET_TIME_UNIT_FOREVER_REL);
731 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
732 make_download_status (&pi, dc);
733 pi.value.download.specifics.start.meta = meta;
734 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
742 * Free entries in the map.
744 * @param cls unused (NULL)
746 * @param entry entry of type "struct DownloadRequest" which is freed
750 free_entry (void *cls,
751 const GNUNET_HashCode *key,
760 * Stop a download (aborts if download is incomplete).
762 * @param dc handle for the download
763 * @param do_delete delete files of incomplete downloads
766 GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
769 struct GNUNET_FS_ProgressInfo pi;
771 // FIXME: make unpersistent
772 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
773 make_download_status (&pi, dc);
774 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
777 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
778 GNUNET_SCHEDULER_cancel (dc->h->sched,
780 if (NULL != dc->client)
781 GNUNET_CLIENT_disconnect (dc->client);
782 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
785 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
786 if (dc->filename != NULL)
788 if (NULL != dc->handle)
789 GNUNET_DISK_file_close (dc->handle);
790 if ( (dc->completed != dc->length) &&
791 (GNUNET_YES == do_delete) )
793 if (0 != UNLINK (dc->filename))
794 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
798 GNUNET_free (dc->filename);
800 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
801 GNUNET_FS_uri_destroy (dc->uri);
811 * Check if self block is already present on the drive. If the block
812 * is a dblock and present, the ProgressModel is notified. If the
813 * block is present and it is an iblock, downloading the children is
816 * Also checks if the block is within the range of blocks
817 * that we are supposed to download. If not, the method
818 * returns as if the block is present but does NOT signal
821 * @param node that is checked for presence
822 * @return GNUNET_YES if present, GNUNET_NO if not.
825 check_node_present (const struct Node *node)
833 size = get_node_size (node);
834 /* first check if node is within range.
835 For now, keeping it simple, we only do
836 this for level-0 nodes */
837 if ((node->level == 0) &&
838 ((node->offset + size < node->ctx->offset) ||
839 (node->offset >= node->ctx->offset + node->ctx->length)))
841 data = GNUNET_malloc (size);
843 res = read_from_files (node->ctx, node->level, node->offset, data, size);
846 GNUNET_hash (data, size, &hc);
847 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
849 notify_client_about_progress (node, data, size);
851 iblock_download_children (node, data, size);
862 /* end of fs_download.c */