2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - location URI suppport (can wait, easy)
27 * - check if blocks exist already (can wait, easy)
28 * - handle recursive downloads (need directory &
29 * fs-level download-parallelism management, can wait)
30 * - check if iblocks can be computed from existing blocks (can wait, hard)
31 * - persistence (can wait)
34 #include "gnunet_constants.h"
35 #include "gnunet_fs_service.h"
39 #define DEBUG_DOWNLOAD GNUNET_YES
42 * We're storing the IBLOCKS after the
43 * DBLOCKS on disk (so that we only have
44 * to truncate the file once we're done).
46 * Given the offset of a block (with respect
47 * to the DBLOCKS) and its depth, return the
48 * offset where we would store this block
52 * @param fsize overall file size
53 * @param off offset of the block in the file
54 * @param depth depth of the block in the tree
55 * @param treedepth maximum depth of the tree
56 * @return off for DBLOCKS (depth == treedepth),
57 * otherwise an offset past the end
58 * of the file that does not overlap
59 * with the range for any other block
62 compute_disk_offset (uint64_t fsize,
65 unsigned int treedepth)
68 uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
69 uint64_t loff; /* where do IBlocks for level "i" start? */
70 unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
72 if (depth == treedepth)
74 /* first IBlocks start at the end of file, rounded up
75 to full DBLOCK_SIZE */
76 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
77 lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
78 GNUNET_assert (0 == (off % DBLOCK_SIZE));
79 ioff = (off / DBLOCK_SIZE);
80 for (i=treedepth-1;i>depth;i--)
83 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
84 GNUNET_assert (lsize > 0);
85 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
86 ioff /= CHK_PER_INODE;
88 return loff + ioff * sizeof (struct ContentHashKey);
93 * Given a file of the specified treedepth and a block at the given
94 * offset and depth, calculate the offset for the CHK at the given
97 * @param offset the offset of the first
98 * DBLOCK in the subtree of the
100 * @param depth the depth of the IBLOCK in the tree
101 * @param treedepth overall depth of the tree
102 * @param k which CHK in the IBLOCK are we
104 * @return offset if k=0, otherwise an appropriately
105 * larger value (i.e., if depth = treedepth-1,
106 * the returned value should be offset+DBLOCK_SIZE)
109 compute_dblock_offset (uint64_t offset,
111 unsigned int treedepth,
115 uint64_t lsize; /* what is the size of the sum of all DBlocks
116 that a CHK at level i corresponds to? */
118 if (depth == treedepth)
121 for (i=treedepth-1;i>depth;i--)
122 lsize *= CHK_PER_INODE;
123 return offset + i * lsize;
128 * Fill in all of the generic fields for
131 * @param pi structure to fill in
132 * @param dc overall download context
135 make_download_status (struct GNUNET_FS_ProgressInfo *pi,
136 struct GNUNET_FS_DownloadContext *dc)
138 pi->value.download.dc = dc;
139 pi->value.download.cctx
141 pi->value.download.pctx
142 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
143 pi->value.download.uri
145 pi->value.download.filename
147 pi->value.download.size
149 pi->value.download.duration
150 = GNUNET_TIME_absolute_get_duration (dc->start_time);
151 pi->value.download.completed
153 pi->value.download.anonymity
159 * Schedule the download of the specified
162 * @param dc overall download this block belongs to
163 * @param chk content-hash-key of the block
164 * @param offset offset of the block in the file
165 * (for IBlocks, the offset is the lowest
166 * offset of any DBlock in the subtree under
168 * @param depth depth of the block, 0 is the root of the tree
171 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
172 const struct ContentHashKey *chk,
176 struct DownloadRequest *sm;
180 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
181 "Scheduling download at offset %llu and depth %u for `%s'\n",
182 (unsigned long long) offset,
184 GNUNET_h2s (&chk->query));
186 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
190 if ( (dc->old_file_size > off) &&
191 (dc->handle != NULL) &&
193 GNUNET_DISK_file_seek (dc->handle,
195 GNUNET_DISK_SEEK_SET) ) )
197 // FIXME: check if block exists on disk!
198 // (read block, encode, compare with
199 // query; if matches, simply return)
201 if (depth < dc->treedepth)
203 // FIXME: try if we could
204 // reconstitute this IBLOCK
205 // from the existing blocks on disk (can wait)
206 // (read block(s), encode, compare with
207 // query; if matches, simply return)
209 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
213 sm->is_pending = GNUNET_YES;
214 sm->next = dc->pending;
216 GNUNET_CONTAINER_multihashmap_put (dc->active,
219 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
224 * We've lost our connection with the FS service.
225 * Re-establish it and re-transmit all of our
228 * @param dc download context that is having trouble
231 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
235 * Compute how many bytes of data should be stored in
236 * the specified node.
238 * @param fsize overall file size
239 * @param totaldepth depth of the entire tree
240 * @param offset offset of the node
241 * @param depth depth of the node
242 * @return number of bytes stored in this node
245 calculate_block_size (uint64_t fsize,
246 unsigned int totaldepth,
256 GNUNET_assert (offset < fsize);
257 if (depth == totaldepth)
260 if (offset + ret > fsize)
261 ret = (size_t) (fsize - offset);
266 for (i = totaldepth-1; i > depth; i--)
267 rsize *= CHK_PER_INODE;
268 epos = offset + rsize * CHK_PER_INODE;
269 GNUNET_assert (epos > offset);
272 /* round up when computing #CHKs in our IBlock */
273 chks = (epos - offset + rsize - 1) / rsize;
274 GNUNET_assert (chks <= CHK_PER_INODE);
275 return chks * sizeof (struct ContentHashKey);
280 * Process a download result.
282 * @param dc our download context
283 * @param type type of the result
284 * @param data the (encrypted) response
285 * @param size size of data
288 process_result (struct GNUNET_FS_DownloadContext *dc,
293 struct GNUNET_FS_ProgressInfo pi;
294 GNUNET_HashCode query;
295 struct DownloadRequest *sm;
296 struct GNUNET_CRYPTO_AesSessionKey skey;
297 struct GNUNET_CRYPTO_AesInitializationVector iv;
302 struct ContentHashKey *chk;
305 GNUNET_CRYPTO_hash (data, size, &query);
306 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
313 if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
318 dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
320 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
321 make_download_status (&pi, dc);
322 pi.value.download.specifics.error.message = dc->emsg;
323 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
325 /* abort all pending requests */
326 GNUNET_CLIENT_disconnect (dc->client);
330 GNUNET_assert (GNUNET_YES ==
331 GNUNET_CONTAINER_multihashmap_remove (dc->active,
334 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
335 GNUNET_CRYPTO_aes_decrypt (data,
341 if ( (NULL != dc->handle) &&
342 ( (sm->depth == dc->treedepth) ||
343 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
345 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
351 GNUNET_DISK_file_seek (dc->handle,
353 GNUNET_DISK_SEEK_SET) ) )
354 GNUNET_asprintf (&emsg,
355 _("Failed to seek to offset %llu in file `%s': %s\n"),
356 (unsigned long long) off,
360 GNUNET_DISK_file_write (dc->handle,
363 GNUNET_asprintf (&emsg,
364 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
366 (unsigned long long) off,
372 // FIXME: make persistent
375 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
376 make_download_status (&pi, dc);
377 pi.value.download.specifics.error.message = emsg;
378 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
381 /* abort all pending requests */
382 GNUNET_CLIENT_disconnect (dc->client);
387 if (sm->depth == dc->treedepth)
390 if (sm->offset < dc->offset)
392 /* starting offset begins in the middle of pt,
393 do not count first bytes as progress */
394 GNUNET_assert (app > (dc->offset - sm->offset));
395 app -= (dc->offset - sm->offset);
397 if (sm->offset + size > dc->offset + dc->length)
399 /* end of block is after relevant range,
400 do not count last bytes as progress */
401 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
402 app -= (sm->offset + size) - (dc->offset + dc->length);
404 dc->completed += app;
407 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
408 make_download_status (&pi, dc);
409 pi.value.download.specifics.progress.data = pt;
410 pi.value.download.specifics.progress.offset = sm->offset;
411 pi.value.download.specifics.progress.data_len = size;
412 pi.value.download.specifics.progress.depth = sm->depth;
413 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
415 GNUNET_assert (dc->completed <= dc->length);
416 if (dc->completed == dc->length)
418 /* truncate file to size (since we store IBlocks at the end) */
419 if (dc->handle != NULL)
421 GNUNET_DISK_file_close (dc->handle);
423 if (0 != truncate (dc->filename,
424 GNUNET_ntohll (dc->uri->data.chk.file_length)))
425 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
429 /* signal completion */
430 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
431 make_download_status (&pi, dc);
432 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
434 GNUNET_assert (sm->depth == dc->treedepth);
436 // FIXME: make persistent
437 if (sm->depth == dc->treedepth)
439 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
440 chk = (struct ContentHashKey*) pt;
441 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
443 off = compute_dblock_offset (sm->offset,
447 if ( (off + DBLOCK_SIZE >= dc->offset) &&
448 (off < dc->offset + dc->length) )
449 schedule_block_download (dc,
458 * Type of a function to call when we receive a message
462 * @param msg message received, NULL on timeout or fatal error
465 receive_results (void *cls,
466 const struct GNUNET_MessageHeader * msg)
468 struct GNUNET_FS_DownloadContext *dc = cls;
469 const struct ContentMessage *cm;
472 if ( (NULL == msg) ||
473 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
474 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
479 msize = ntohs (msg->size);
480 cm = (const struct ContentMessage*) msg;
484 msize - sizeof (struct ContentMessage));
485 /* continue receiving */
486 GNUNET_CLIENT_receive (dc->client,
489 GNUNET_TIME_UNIT_FOREVER_REL);
495 * We're ready to transmit a search request to the
496 * file-sharing service. Do it. If there is
497 * more than one request pending, try to send
498 * multiple or request another transmission.
501 * @param size number of bytes available in buf
502 * @param buf where the callee should write the message
503 * @return number of bytes written to buf
506 transmit_download_request (void *cls,
510 struct GNUNET_FS_DownloadContext *dc = cls;
512 struct SearchMessage *sm;
519 GNUNET_assert (size >= sizeof (struct SearchMessage));
522 while ( (dc->pending == NULL) &&
523 (size > msize + sizeof (struct SearchMessage)) )
525 memset (sm, 0, sizeof (struct SearchMessage));
526 sm->header.size = htons (sizeof (struct SearchMessage));
527 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
528 sm->anonymity_level = htonl (dc->anonymity);
529 sm->target = dc->target.hashPubKey;
530 sm->query = dc->pending->chk.query;
531 dc->pending->is_pending = GNUNET_NO;
532 dc->pending = dc->pending->next;
533 msize += sizeof (struct SearchMessage);
541 * Reconnect to the FS service and transmit
544 * @param cls our download context
548 do_reconnect (void *cls,
549 const struct GNUNET_SCHEDULER_TaskContext *tc)
551 struct GNUNET_FS_DownloadContext *dc = cls;
552 struct GNUNET_CLIENT_Connection *client;
554 dc->task = GNUNET_SCHEDULER_NO_TASK;
555 client = GNUNET_CLIENT_connect (dc->h->sched,
564 GNUNET_CLIENT_notify_transmit_ready (client,
565 sizeof (struct SearchMessage),
566 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
567 &transmit_download_request,
569 GNUNET_CLIENT_receive (client,
572 GNUNET_TIME_UNIT_FOREVER_REL);
577 * Add entries that are not yet pending back to
580 * @param cls our download context
582 * @param entry entry of type "struct DownloadRequest"
586 retry_entry (void *cls,
587 const GNUNET_HashCode *key,
590 struct GNUNET_FS_DownloadContext *dc = cls;
591 struct DownloadRequest *dr = entry;
593 if (! dr->is_pending)
595 dr->next = dc->pending;
596 dr->is_pending = GNUNET_YES;
604 * We've lost our connection with the FS service.
605 * Re-establish it and re-transmit all of our
608 * @param dc download context that is having trouble
611 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
614 if (NULL != dc->client)
616 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
619 GNUNET_CLIENT_disconnect (dc->client);
623 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
625 GNUNET_SCHEDULER_PRIORITY_IDLE,
626 GNUNET_SCHEDULER_NO_TASK,
627 GNUNET_TIME_UNIT_SECONDS,
634 * Download parts of a file. Note that this will store
635 * the blocks at the respective offset in the given file. Also, the
636 * download is still using the blocking of the underlying FS
637 * encoding. As a result, the download may *write* outside of the
638 * given boundaries (if offset and length do not match the 32k FS
639 * block boundaries). <p>
641 * This function should be used to focus a download towards a
642 * particular portion of the file (optimization), not to strictly
643 * limit the download to exactly those bytes.
645 * @param h handle to the file sharing subsystem
646 * @param uri the URI of the file (determines what to download); CHK or LOC URI
647 * @param meta known metadata for the file (can be NULL)
648 * @param filename where to store the file, maybe NULL (then no file is
649 * created on disk and data must be grabbed from the callbacks)
650 * @param offset at what offset should we start the download (typically 0)
651 * @param length how many bytes should be downloaded starting at offset
652 * @param anonymity anonymity level to use for the download
653 * @param options various options
654 * @param parent parent download to associate this download with (use NULL
655 * for top-level downloads; useful for manually-triggered recursive downloads)
656 * @return context that can be used to control this download
658 struct GNUNET_FS_DownloadContext *
659 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
660 const struct GNUNET_FS_Uri *uri,
661 const struct GNUNET_CONTAINER_MetaData *meta,
662 const char *filename,
666 enum GNUNET_FS_DownloadOptions options,
667 struct GNUNET_FS_DownloadContext *parent)
669 struct GNUNET_FS_ProgressInfo pi;
670 struct GNUNET_FS_DownloadContext *dc;
671 struct GNUNET_CLIENT_Connection *client;
673 client = GNUNET_CLIENT_connect (h->sched,
678 // FIXME: add support for "loc" URIs!
679 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
680 if ( (offset + length < offset) ||
681 (offset + length > uri->data.chk.file_length) )
687 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
688 "Starting download `%s' of %llu bytes\n",
690 (unsigned long long) length);
692 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
696 dc->uri = GNUNET_FS_uri_dup (uri);
697 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
698 if (NULL != filename)
700 dc->filename = GNUNET_strdup (filename);
701 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
702 GNUNET_DISK_file_size (filename,
705 dc->handle = GNUNET_DISK_file_open (filename,
706 GNUNET_DISK_OPEN_READWRITE |
707 GNUNET_DISK_OPEN_CREATE,
708 GNUNET_DISK_PERM_USER_READ |
709 GNUNET_DISK_PERM_USER_WRITE |
710 GNUNET_DISK_PERM_GROUP_READ |
711 GNUNET_DISK_PERM_OTHER_READ);
712 if (dc->handle == NULL)
714 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
715 _("Download failed: could not open file `%s': %s\n"),
718 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
719 GNUNET_FS_uri_destroy (dc->uri);
720 GNUNET_free (dc->filename);
721 GNUNET_CLIENT_disconnect (dc->client);
726 // FIXME: set "dc->target" for LOC uris!
729 dc->anonymity = anonymity;
730 dc->options = options;
731 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
732 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
734 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
735 "Download tree has depth %u\n",
738 // FIXME: make persistent
739 schedule_block_download (dc,
740 &dc->uri->data.chk.chk,
743 GNUNET_CLIENT_notify_transmit_ready (client,
744 sizeof (struct SearchMessage),
745 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
746 &transmit_download_request,
748 GNUNET_CLIENT_receive (client,
751 GNUNET_TIME_UNIT_FOREVER_REL);
752 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
753 make_download_status (&pi, dc);
754 pi.value.download.specifics.start.meta = meta;
755 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
763 * Free entries in the map.
765 * @param cls unused (NULL)
767 * @param entry entry of type "struct DownloadRequest" which is freed
771 free_entry (void *cls,
772 const GNUNET_HashCode *key,
781 * Stop a download (aborts if download is incomplete).
783 * @param dc handle for the download
784 * @param do_delete delete files of incomplete downloads
787 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
790 struct GNUNET_FS_ProgressInfo pi;
792 // FIXME: make unpersistent
793 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
794 make_download_status (&pi, dc);
795 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
798 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
799 GNUNET_SCHEDULER_cancel (dc->h->sched,
801 if (NULL != dc->client)
802 GNUNET_CLIENT_disconnect (dc->client);
803 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
806 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
807 if (dc->filename != NULL)
809 if (NULL != dc->handle)
810 GNUNET_DISK_file_close (dc->handle);
811 if ( (dc->completed != dc->length) &&
812 (GNUNET_YES == do_delete) )
814 if (0 != UNLINK (dc->filename))
815 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
819 GNUNET_free (dc->filename);
821 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
822 GNUNET_FS_uri_destroy (dc->uri);
832 * Check if self block is already present on the drive. If the block
833 * is a dblock and present, the ProgressModel is notified. If the
834 * block is present and it is an iblock, downloading the children is
837 * Also checks if the block is within the range of blocks
838 * that we are supposed to download. If not, the method
839 * returns as if the block is present but does NOT signal
842 * @param node that is checked for presence
843 * @return GNUNET_YES if present, GNUNET_NO if not.
846 check_node_present (const struct Node *node)
854 size = get_node_size (node);
855 /* first check if node is within range.
856 For now, keeping it simple, we only do
857 this for level-0 nodes */
858 if ((node->level == 0) &&
859 ((node->offset + size < node->ctx->offset) ||
860 (node->offset >= node->ctx->offset + node->ctx->length)))
862 data = GNUNET_malloc (size);
864 res = read_from_files (node->ctx, node->level, node->offset, data, size);
867 GNUNET_hash (data, size, &hc);
868 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
870 notify_client_about_progress (node, data, size);
872 iblock_download_children (node, data, size);
883 /* end of fs_download.c */