2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - location URI suppport (can wait, easy)
27 * - check if blocks exist already (can wait, easy)
28 * - handle recursive downloads (need directory &
29 * fs-level download-parallelism management, can wait)
30 * - check if iblocks can be computed from existing blocks (can wait, hard)
31 * - persistence (can wait)
34 #include "gnunet_constants.h"
35 #include "gnunet_fs_service.h"
39 #define DEBUG_DOWNLOAD GNUNET_YES
42 * We're storing the IBLOCKS after the
43 * DBLOCKS on disk (so that we only have
44 * to truncate the file once we're done).
46 * Given the offset of a block (with respect
47 * to the DBLOCKS) and its depth, return the
48 * offset where we would store this block
52 * @param fsize overall file size
53 * @param off offset of the block in the file
54 * @param depth depth of the block in the tree
55 * @param treedepth maximum depth of the tree
56 * @return off for DBLOCKS (depth == treedepth),
57 * otherwise an offset past the end
58 * of the file that does not overlap
59 * with the range for any other block
62 compute_disk_offset (uint64_t fsize,
65 unsigned int treedepth)
68 uint64_t lsize; /* what is the size of all IBlocks for level "i"? */
69 uint64_t loff; /* where do IBlocks for level "i" start? */
70 unsigned int ioff; /* which IBlock corresponds to "off" at level "i"? */
72 if (depth == treedepth)
74 /* first IBlocks start at the end of file, rounded up
75 to full DBLOCK_SIZE */
76 loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
77 lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
78 GNUNET_assert (0 == (off % DBLOCK_SIZE));
79 ioff = (off / DBLOCK_SIZE);
80 for (i=treedepth-1;i>depth;i--)
83 lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
84 GNUNET_assert (lsize > 0);
85 GNUNET_assert (0 == (ioff % CHK_PER_INODE));
86 ioff /= CHK_PER_INODE;
88 return loff + ioff * sizeof (struct ContentHashKey);
93 * Given a file of the specified treedepth and a block at the given
94 * offset and depth, calculate the offset for the CHK at the given
97 * @param offset the offset of the first
98 * DBLOCK in the subtree of the
100 * @param depth the depth of the IBLOCK in the tree
101 * @param treedepth overall depth of the tree
102 * @param k which CHK in the IBLOCK are we
104 * @return offset if k=0, otherwise an appropriately
105 * larger value (i.e., if depth = treedepth-1,
106 * the returned value should be offset+DBLOCK_SIZE)
109 compute_dblock_offset (uint64_t offset,
111 unsigned int treedepth,
115 uint64_t lsize; /* what is the size of the sum of all DBlocks
116 that a CHK at level i corresponds to? */
118 if (depth == treedepth)
121 for (i=treedepth-1;i>depth;i--)
122 lsize *= CHK_PER_INODE;
123 return offset + i * lsize;
128 * Fill in all of the generic fields for
131 * @param pi structure to fill in
132 * @param dc overall download context
135 make_download_status (struct GNUNET_FS_ProgressInfo *pi,
136 struct GNUNET_FS_DownloadContext *dc)
138 pi->value.download.dc = dc;
139 pi->value.download.cctx
141 pi->value.download.pctx
142 = (dc->parent == NULL) ? NULL : dc->parent->client_info;
143 pi->value.download.uri
145 pi->value.download.filename
147 pi->value.download.size
149 pi->value.download.duration
150 = GNUNET_TIME_absolute_get_duration (dc->start_time);
151 pi->value.download.completed
153 pi->value.download.anonymity
159 * Schedule the download of the specified
162 * @param dc overall download this block belongs to
163 * @param chk content-hash-key of the block
164 * @param offset offset of the block in the file
165 * (for IBlocks, the offset is the lowest
166 * offset of any DBlock in the subtree under
168 * @param depth depth of the block, 0 is the root of the tree
171 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
172 const struct ContentHashKey *chk,
176 struct DownloadRequest *sm;
179 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
183 if ( (dc->old_file_size > off) &&
184 (dc->handle != NULL) &&
186 GNUNET_DISK_file_seek (dc->handle,
188 GNUNET_DISK_SEEK_SET) ) )
190 // FIXME: check if block exists on disk!
191 // (read block, encode, compare with
192 // query; if matches, simply return)
194 if (depth < dc->treedepth)
196 // FIXME: try if we could
197 // reconstitute this IBLOCK
198 // from the existing blocks on disk (can wait)
199 // (read block(s), encode, compare with
200 // query; if matches, simply return)
202 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
206 sm->is_pending = GNUNET_YES;
207 sm->next = dc->pending;
209 GNUNET_CONTAINER_multihashmap_put (dc->active,
212 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
217 * We've lost our connection with the FS service.
218 * Re-establish it and re-transmit all of our
221 * @param dc download context that is having trouble
224 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
228 * Compute how many bytes of data should be stored in
229 * the specified node.
231 * @param fsize overall file size
232 * @param totaldepth depth of the entire tree
233 * @param offset offset of the node
234 * @param depth depth of the node
235 * @return number of bytes stored in this node
238 calculate_block_size (uint64_t fsize,
239 unsigned int totaldepth,
249 GNUNET_assert (offset < fsize);
250 if (depth == totaldepth)
253 if (offset + ret > fsize)
254 ret = (size_t) (fsize - offset);
259 for (i = totaldepth-1; i > depth; i--)
260 rsize *= CHK_PER_INODE;
261 epos = offset + rsize * CHK_PER_INODE;
262 GNUNET_assert (epos > offset);
265 /* round up when computing #CHKs in our IBlock */
266 chks = (epos - offset + rsize - 1) / rsize;
267 GNUNET_assert (chks <= CHK_PER_INODE);
268 return chks * sizeof (struct ContentHashKey);
273 * Process a download result.
275 * @param dc our download context
276 * @param type type of the result
277 * @param data the (encrypted) response
278 * @param size size of data
281 process_result (struct GNUNET_FS_DownloadContext *dc,
286 struct GNUNET_FS_ProgressInfo pi;
287 GNUNET_HashCode query;
288 struct DownloadRequest *sm;
289 struct GNUNET_CRYPTO_AesSessionKey skey;
290 struct GNUNET_CRYPTO_AesInitializationVector iv;
295 struct ContentHashKey *chk;
298 GNUNET_CRYPTO_hash (data, size, &query);
299 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
306 if (size != calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
311 dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
313 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
314 make_download_status (&pi, dc);
315 pi.value.download.specifics.error.message = dc->emsg;
316 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
318 /* abort all pending requests */
319 GNUNET_CLIENT_disconnect (dc->client);
323 GNUNET_assert (GNUNET_YES ==
324 GNUNET_CONTAINER_multihashmap_remove (dc->active,
327 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
328 GNUNET_CRYPTO_aes_decrypt (data,
334 if ( (NULL != dc->handle) &&
335 ( (sm->depth == dc->treedepth) ||
336 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
338 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
344 GNUNET_DISK_file_seek (dc->handle,
346 GNUNET_DISK_SEEK_SET) ) )
347 GNUNET_asprintf (&emsg,
348 _("Failed to seek to offset %llu in file `%s': %s\n"),
349 (unsigned long long) off,
353 GNUNET_DISK_file_write (dc->handle,
356 GNUNET_asprintf (&emsg,
357 _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
359 (unsigned long long) off,
365 // FIXME: make persistent
368 pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
369 make_download_status (&pi, dc);
370 pi.value.download.specifics.error.message = emsg;
371 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
374 /* abort all pending requests */
375 GNUNET_CLIENT_disconnect (dc->client);
380 if (sm->depth == dc->treedepth)
383 if (sm->offset < dc->offset)
385 /* starting offset begins in the middle of pt,
386 do not count first bytes as progress */
387 GNUNET_assert (app > (dc->offset - sm->offset));
388 app -= (dc->offset - sm->offset);
390 if (sm->offset + size > dc->offset + dc->length)
392 /* end of block is after relevant range,
393 do not count last bytes as progress */
394 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
395 app -= (sm->offset + size) - (dc->offset + dc->length);
397 dc->completed += app;
400 pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
401 make_download_status (&pi, dc);
402 pi.value.download.specifics.progress.data = pt;
403 pi.value.download.specifics.progress.offset = sm->offset;
404 pi.value.download.specifics.progress.data_len = size;
405 pi.value.download.specifics.progress.depth = sm->depth;
406 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
408 GNUNET_assert (dc->completed <= dc->length);
409 if (dc->completed == dc->length)
411 /* truncate file to size (since we store IBlocks at the end) */
412 if (dc->handle != NULL)
414 GNUNET_DISK_file_close (dc->handle);
416 if (0 != truncate (dc->filename,
417 GNUNET_ntohll (dc->uri->data.chk.file_length)))
418 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
422 /* signal completion */
423 pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
424 make_download_status (&pi, dc);
425 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
427 GNUNET_assert (sm->depth == dc->treedepth);
429 // FIXME: make persistent
430 if (sm->depth == dc->treedepth)
432 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
433 chk = (struct ContentHashKey*) pt;
434 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
436 off = compute_dblock_offset (sm->offset,
440 if ( (off + DBLOCK_SIZE >= dc->offset) &&
441 (off < dc->offset + dc->length) )
442 schedule_block_download (dc,
451 * Type of a function to call when we receive a message
455 * @param msg message received, NULL on timeout or fatal error
458 receive_results (void *cls,
459 const struct GNUNET_MessageHeader * msg)
461 struct GNUNET_FS_DownloadContext *dc = cls;
462 const struct ContentMessage *cm;
465 if ( (NULL == msg) ||
466 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
467 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
472 msize = ntohs (msg->size);
473 cm = (const struct ContentMessage*) msg;
477 msize - sizeof (struct ContentMessage));
478 /* continue receiving */
479 GNUNET_CLIENT_receive (dc->client,
482 GNUNET_TIME_UNIT_FOREVER_REL);
488 * We're ready to transmit a search request to the
489 * file-sharing service. Do it. If there is
490 * more than one request pending, try to send
491 * multiple or request another transmission.
494 * @param size number of bytes available in buf
495 * @param buf where the callee should write the message
496 * @return number of bytes written to buf
499 transmit_download_request (void *cls,
503 struct GNUNET_FS_DownloadContext *dc = cls;
505 struct SearchMessage *sm;
512 GNUNET_assert (size >= sizeof (struct SearchMessage));
515 while ( (dc->pending == NULL) &&
516 (size > msize + sizeof (struct SearchMessage)) )
518 memset (sm, 0, sizeof (struct SearchMessage));
519 sm->header.size = htons (sizeof (struct SearchMessage));
520 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
521 sm->anonymity_level = htonl (dc->anonymity);
522 sm->target = dc->target.hashPubKey;
523 sm->query = dc->pending->chk.query;
524 dc->pending->is_pending = GNUNET_NO;
525 dc->pending = dc->pending->next;
526 msize += sizeof (struct SearchMessage);
534 * Reconnect to the FS service and transmit
537 * @param cls our download context
541 do_reconnect (void *cls,
542 const struct GNUNET_SCHEDULER_TaskContext *tc)
544 struct GNUNET_FS_DownloadContext *dc = cls;
545 struct GNUNET_CLIENT_Connection *client;
547 dc->task = GNUNET_SCHEDULER_NO_TASK;
548 client = GNUNET_CLIENT_connect (dc->h->sched,
557 GNUNET_CLIENT_notify_transmit_ready (client,
558 sizeof (struct SearchMessage),
559 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
560 &transmit_download_request,
562 GNUNET_CLIENT_receive (client,
565 GNUNET_TIME_UNIT_FOREVER_REL);
570 * Add entries that are not yet pending back to
573 * @param cls our download context
575 * @param entry entry of type "struct DownloadRequest"
579 retry_entry (void *cls,
580 const GNUNET_HashCode *key,
583 struct GNUNET_FS_DownloadContext *dc = cls;
584 struct DownloadRequest *dr = entry;
586 if (! dr->is_pending)
588 dr->next = dc->pending;
589 dr->is_pending = GNUNET_YES;
597 * We've lost our connection with the FS service.
598 * Re-establish it and re-transmit all of our
601 * @param dc download context that is having trouble
604 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
607 if (NULL != dc->client)
609 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
612 GNUNET_CLIENT_disconnect (dc->client);
616 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
618 GNUNET_SCHEDULER_PRIORITY_IDLE,
619 GNUNET_SCHEDULER_NO_TASK,
620 GNUNET_TIME_UNIT_SECONDS,
627 * Download parts of a file. Note that this will store
628 * the blocks at the respective offset in the given file. Also, the
629 * download is still using the blocking of the underlying FS
630 * encoding. As a result, the download may *write* outside of the
631 * given boundaries (if offset and length do not match the 32k FS
632 * block boundaries). <p>
634 * This function should be used to focus a download towards a
635 * particular portion of the file (optimization), not to strictly
636 * limit the download to exactly those bytes.
638 * @param h handle to the file sharing subsystem
639 * @param uri the URI of the file (determines what to download); CHK or LOC URI
640 * @param meta known metadata for the file (can be NULL)
641 * @param filename where to store the file, maybe NULL (then no file is
642 * created on disk and data must be grabbed from the callbacks)
643 * @param offset at what offset should we start the download (typically 0)
644 * @param length how many bytes should be downloaded starting at offset
645 * @param anonymity anonymity level to use for the download
646 * @param options various options
647 * @param parent parent download to associate this download with (use NULL
648 * for top-level downloads; useful for manually-triggered recursive downloads)
649 * @return context that can be used to control this download
651 struct GNUNET_FS_DownloadContext *
652 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
653 const struct GNUNET_FS_Uri *uri,
654 const struct GNUNET_CONTAINER_MetaData *meta,
655 const char *filename,
659 enum GNUNET_FS_DownloadOptions options,
660 struct GNUNET_FS_DownloadContext *parent)
662 struct GNUNET_FS_ProgressInfo pi;
663 struct GNUNET_FS_DownloadContext *dc;
664 struct GNUNET_CLIENT_Connection *client;
666 client = GNUNET_CLIENT_connect (h->sched,
671 // FIXME: add support for "loc" URIs!
672 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
673 if ( (offset + length < offset) ||
674 (offset + length > uri->data.chk.file_length) )
679 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
683 dc->uri = GNUNET_FS_uri_dup (uri);
684 dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
685 if (NULL != filename)
687 dc->filename = GNUNET_strdup (filename);
688 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
689 GNUNET_DISK_file_size (filename,
692 dc->handle = GNUNET_DISK_file_open (filename,
693 GNUNET_DISK_OPEN_READWRITE |
694 GNUNET_DISK_OPEN_CREATE,
695 GNUNET_DISK_PERM_USER_READ |
696 GNUNET_DISK_PERM_USER_WRITE |
697 GNUNET_DISK_PERM_GROUP_READ |
698 GNUNET_DISK_PERM_OTHER_READ);
699 if (dc->handle == NULL)
701 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
702 _("Download failed: could not open file `%s': %s\n"),
705 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
706 GNUNET_FS_uri_destroy (dc->uri);
707 GNUNET_free (dc->filename);
708 GNUNET_CLIENT_disconnect (dc->client);
713 // FIXME: set "dc->target" for LOC uris!
716 dc->anonymity = anonymity;
717 dc->options = options;
718 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
719 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
720 // FIXME: make persistent
721 schedule_block_download (dc,
722 &dc->uri->data.chk.chk,
725 GNUNET_CLIENT_notify_transmit_ready (client,
726 sizeof (struct SearchMessage),
727 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
728 &transmit_download_request,
730 GNUNET_CLIENT_receive (client,
733 GNUNET_TIME_UNIT_FOREVER_REL);
734 pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
735 make_download_status (&pi, dc);
736 pi.value.download.specifics.start.meta = meta;
737 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
745 * Free entries in the map.
747 * @param cls unused (NULL)
749 * @param entry entry of type "struct DownloadRequest" which is freed
753 free_entry (void *cls,
754 const GNUNET_HashCode *key,
763 * Stop a download (aborts if download is incomplete).
765 * @param dc handle for the download
766 * @param do_delete delete files of incomplete downloads
769 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
772 struct GNUNET_FS_ProgressInfo pi;
774 // FIXME: make unpersistent
775 pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
776 make_download_status (&pi, dc);
777 dc->client_info = dc->h->upcb (dc->h->upcb_cls,
780 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
781 GNUNET_SCHEDULER_cancel (dc->h->sched,
783 if (NULL != dc->client)
784 GNUNET_CLIENT_disconnect (dc->client);
785 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
788 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
789 if (dc->filename != NULL)
791 if (NULL != dc->handle)
792 GNUNET_DISK_file_close (dc->handle);
793 if ( (dc->completed != dc->length) &&
794 (GNUNET_YES == do_delete) )
796 if (0 != UNLINK (dc->filename))
797 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
801 GNUNET_free (dc->filename);
803 GNUNET_CONTAINER_meta_data_destroy (dc->meta);
804 GNUNET_FS_uri_destroy (dc->uri);
814 * Check if self block is already present on the drive. If the block
815 * is a dblock and present, the ProgressModel is notified. If the
816 * block is present and it is an iblock, downloading the children is
819 * Also checks if the block is within the range of blocks
820 * that we are supposed to download. If not, the method
821 * returns as if the block is present but does NOT signal
824 * @param node that is checked for presence
825 * @return GNUNET_YES if present, GNUNET_NO if not.
828 check_node_present (const struct Node *node)
836 size = get_node_size (node);
837 /* first check if node is within range.
838 For now, keeping it simple, we only do
839 this for level-0 nodes */
840 if ((node->level == 0) &&
841 ((node->offset + size < node->ctx->offset) ||
842 (node->offset >= node->ctx->offset + node->ctx->length)))
844 data = GNUNET_malloc (size);
846 res = read_from_files (node->ctx, node->level, node->offset, data, size);
849 GNUNET_hash (data, size, &hc);
850 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
852 notify_client_about_progress (node, data, size);
854 iblock_download_children (node, data, size);
865 /* end of fs_download.c */