2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fs/fs_download.c
22 * @brief download methods
23 * @author Christian Grothoff
26 * - offset calculations
27 * - callback signaling
28 * - check if blocks exist already (can wait)
29 * - location URI suppport (can wait)
30 * - persistence (can wait)
33 #include "gnunet_constants.h"
34 #include "gnunet_fs_service.h"
38 #define DEBUG_DOWNLOAD GNUNET_YES
41 * We're storing the IBLOCKS after the
42 * DBLOCKS on disk (so that we only have
43 * to truncate the file once we're done).
45 * Given the offset of a block (with respect
46 * to the DBLOCKS) and its depth, return the
47 * offset where we would store this block
50 * @param fsize overall file size
51 * @param off offset of the block in the file
52 * @param depth depth of the block in the tree
53 * @param treedepth maximum depth of the tree
54 * @return off for DBLOCKS (depth == treedepth),
55 * otherwise an offset past the end
56 * of the file that does not overlap
57 * with the range for any other block
60 compute_disk_offset (uint64_t fsize,
63 unsigned int treedepth)
65 if (depth == treedepth)
71 * Given a file of the specified treedepth and
72 * a block at the given offset and depth,
73 * calculate the offset for the CHK at
76 * @param offset the offset of the first
77 * DBLOCK in the subtree of the
79 * @param depth the depth of the IBLOCK in the tree
80 * @param treedepth overall depth of the tree
81 * @param i which CHK in the IBLOCK are we
83 * @return offset if i=0, otherwise an appropriately
84 * larger value (i.e., if depth = treedepth-1,
85 * the returned value should be offset+DBLOCK_SIZE)
88 compute_dblock_offset (uint64_t offset,
90 unsigned int treedepth,
93 GNUNET_assert (depth < treedepth);
101 * Schedule the download of the specified
104 * @param dc overall download this block belongs to
105 * @param chk content-hash-key of the block
106 * @param offset offset of the block in the file
107 * (for IBlocks, the offset is the lowest
108 * offset of any DBlock in the subtree under
110 * @param depth depth of the block, 0 is the root of the tree
113 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
114 const struct ContentHashKey *chk,
118 struct DownloadRequest *sm;
121 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
125 if ( (dc->old_file_size > off) &&
126 (dc->handle != NULL) &&
128 GNUNET_DISK_file_seek (dc->handle,
130 GNUNET_DISK_SEEK_SET) ) )
132 // FIXME: check if block exists on disk!
133 // (read block, encode, compare with
134 // query; if matches, simply return)
136 if (depth < dc->treedepth)
138 // FIXME: try if we could
139 // reconstitute this IBLOCK
140 // from the existing blocks on disk (can wait)
141 // (read block(s), encode, compare with
142 // query; if matches, simply return)
144 sm = GNUNET_malloc (sizeof (struct DownloadRequest));
148 sm->is_pending = GNUNET_YES;
149 sm->next = dc->pending;
151 GNUNET_CONTAINER_multihashmap_put (dc->active,
154 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
159 * We've lost our connection with the FS service.
160 * Re-establish it and re-transmit all of our
163 * @param dc download context that is having trouble
166 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
170 * Process a search result.
172 * @param sc our search context
173 * @param type type of the result
174 * @param data the (encrypted) response
175 * @param size size of data
178 process_result (struct GNUNET_FS_DownloadContext *dc,
183 GNUNET_HashCode query;
184 struct DownloadRequest *sm;
185 struct GNUNET_CRYPTO_AesSessionKey skey;
186 struct GNUNET_CRYPTO_AesInitializationVector iv;
191 struct ContentHashKey *chk;
193 // FIXME: check that size is as big as expected, otherwise ignore!!!
194 GNUNET_CRYPTO_hash (data, size, &query);
195 sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
202 GNUNET_assert (GNUNET_YES ==
203 GNUNET_CONTAINER_multihashmap_remove (dc->active,
206 GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
207 GNUNET_CRYPTO_aes_decrypt (data,
213 if ( (NULL != dc->handle) &&
214 ( (sm->depth == dc->treedepth) ||
215 (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
217 off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
221 GNUNET_assert (off !=
222 GNUNET_DISK_file_seek (dc->handle,
224 GNUNET_DISK_SEEK_SET) );
225 GNUNET_DISK_file_write (dc->handle,
229 // FIXME: make persistent
231 if (sm->depth == dc->treedepth)
234 if (sm->offset < dc->offset)
236 /* starting offset begins in the middle of pt,
237 do not count first bytes as progress */
238 GNUNET_assert (app > (dc->offset - sm->offset));
239 app -= (dc->offset - sm->offset);
241 if (sm->offset + size > dc->offset + dc->length)
243 /* end of block is after relevant range,
244 do not count last bytes as progress */
245 GNUNET_assert (app > (sm->offset + size) - (dc->offset + dc->length));
246 app -= (sm->offset + size) - (dc->offset + dc->length);
248 dc->completed += app;
250 // FIXME: call progress callback
251 if (sm->depth == dc->treedepth)
253 GNUNET_assert (0 == (size % sizeof(struct ContentHashKey)));
254 chk = (struct ContentHashKey*) pt;
255 for (i=0;i<(size / sizeof(struct ContentHashKey));i++)
257 off = compute_dblock_offset (sm->offset,
261 if ( (off + DBLOCK_SIZE >= dc->offset) &&
262 (off < dc->offset + dc->length) )
263 schedule_block_download (dc,
272 * Type of a function to call when we receive a message
276 * @param msg message received, NULL on timeout or fatal error
279 receive_results (void *cls,
280 const struct GNUNET_MessageHeader * msg)
282 struct GNUNET_FS_DownloadContext *dc = cls;
283 const struct ContentMessage *cm;
286 if ( (NULL == msg) ||
287 (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
288 (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
293 msize = ntohs (msg->size);
294 cm = (const struct ContentMessage*) msg;
298 msize - sizeof (struct ContentMessage));
299 /* continue receiving */
300 GNUNET_CLIENT_receive (dc->client,
303 GNUNET_TIME_UNIT_FOREVER_REL);
309 * We're ready to transmit a search request to the
310 * file-sharing service. Do it. If there is
311 * more than one request pending, try to send
312 * multiple or request another transmission.
315 * @param size number of bytes available in buf
316 * @param buf where the callee should write the message
317 * @return number of bytes written to buf
320 transmit_download_request (void *cls,
324 struct GNUNET_FS_DownloadContext *dc = cls;
326 struct SearchMessage *sm;
333 GNUNET_assert (size >= sizeof (struct SearchMessage));
336 while ( (dc->pending == NULL) &&
337 (size > msize + sizeof (struct SearchMessage)) )
339 memset (sm, 0, sizeof (struct SearchMessage));
340 sm->header.size = htons (sizeof (struct SearchMessage));
341 sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
342 sm->anonymity_level = htonl (dc->anonymity);
343 sm->target = dc->target.hashPubKey;
344 sm->query = dc->pending->chk.query;
345 dc->pending->is_pending = GNUNET_NO;
346 dc->pending = dc->pending->next;
347 msize += sizeof (struct SearchMessage);
355 * Reconnect to the FS service and transmit
358 * @param cls our download context
362 do_reconnect (void *cls,
363 const struct GNUNET_SCHEDULER_TaskContext *tc)
365 struct GNUNET_FS_DownloadContext *dc = cls;
366 struct GNUNET_CLIENT_Connection *client;
368 dc->task = GNUNET_SCHEDULER_NO_TASK;
369 client = GNUNET_CLIENT_connect (dc->h->sched,
378 GNUNET_CLIENT_notify_transmit_ready (client,
379 sizeof (struct SearchMessage),
380 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
381 &transmit_download_request,
383 GNUNET_CLIENT_receive (client,
386 GNUNET_TIME_UNIT_FOREVER_REL);
391 * Add entries that are not yet pending back to
394 * @param cls our download context
396 * @param entry entry of type "struct DownloadRequest"
400 retry_entry (void *cls,
401 const GNUNET_HashCode *key,
404 struct GNUNET_FS_DownloadContext *dc = cls;
405 struct DownloadRequest *dr = entry;
407 if (! dr->is_pending)
409 dr->next = dc->pending;
410 dr->is_pending = GNUNET_YES;
418 * We've lost our connection with the FS service.
419 * Re-establish it and re-transmit all of our
422 * @param dc download context that is having trouble
425 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
428 if (NULL != dc->client)
430 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
433 GNUNET_CLIENT_disconnect (dc->client);
437 = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
439 GNUNET_SCHEDULER_PRIORITY_IDLE,
440 GNUNET_SCHEDULER_NO_TASK,
441 GNUNET_TIME_UNIT_SECONDS,
448 * Download parts of a file. Note that this will store
449 * the blocks at the respective offset in the given file. Also, the
450 * download is still using the blocking of the underlying FS
451 * encoding. As a result, the download may *write* outside of the
452 * given boundaries (if offset and length do not match the 32k FS
453 * block boundaries). <p>
455 * This function should be used to focus a download towards a
456 * particular portion of the file (optimization), not to strictly
457 * limit the download to exactly those bytes.
459 * @param h handle to the file sharing subsystem
460 * @param uri the URI of the file (determines what to download); CHK or LOC URI
461 * @param filename where to store the file, maybe NULL (then no file is
462 * created on disk and data must be grabbed from the callbacks)
463 * @param offset at what offset should we start the download (typically 0)
464 * @param length how many bytes should be downloaded starting at offset
465 * @param anonymity anonymity level to use for the download
466 * @param options various options
467 * @param parent parent download to associate this download with (use NULL
468 * for top-level downloads; useful for manually-triggered recursive downloads)
469 * @return context that can be used to control this download
471 struct GNUNET_FS_DownloadContext *
472 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
473 const struct GNUNET_FS_Uri *uri,
474 const char *filename,
478 enum GNUNET_FS_DownloadOptions options,
479 struct GNUNET_FS_DownloadContext *parent)
481 struct GNUNET_FS_DownloadContext *dc;
482 struct GNUNET_CLIENT_Connection *client;
484 client = GNUNET_CLIENT_connect (h->sched,
489 // FIXME: add support for "loc" URIs!
490 GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
491 if ( (dc->offset + dc->length < dc->offset) ||
492 (dc->offset + dc->length > uri->data.chk.file_length) )
497 dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
501 dc->uri = GNUNET_FS_uri_dup (uri);
502 if (NULL != filename)
504 dc->filename = GNUNET_strdup (filename);
505 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
506 GNUNET_DISK_file_size (filename,
509 dc->handle = GNUNET_DISK_file_open (filename,
510 GNUNET_DISK_OPEN_READWRITE |
511 GNUNET_DISK_OPEN_CREATE,
512 GNUNET_DISK_PERM_USER_READ |
513 GNUNET_DISK_PERM_USER_WRITE |
514 GNUNET_DISK_PERM_GROUP_READ |
515 GNUNET_DISK_PERM_OTHER_READ);
516 if (dc->handle == NULL)
518 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
519 _("Download failed: could not open file `%s': %s\n"),
522 GNUNET_FS_uri_destroy (dc->uri);
523 GNUNET_free (dc->filename);
524 GNUNET_CLIENT_disconnect (dc->client);
529 // FIXME: set "dc->target" for LOC uris!
532 dc->anonymity = anonymity;
533 dc->options = options;
534 dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / DBLOCK_SIZE));
535 dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
536 // FIXME: make persistent
537 schedule_block_download (dc,
538 &dc->uri->data.chk.chk,
541 GNUNET_CLIENT_notify_transmit_ready (client,
542 sizeof (struct SearchMessage),
543 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
544 &transmit_download_request,
546 GNUNET_CLIENT_receive (client,
549 GNUNET_TIME_UNIT_FOREVER_REL);
550 // FIXME: signal download start
556 * Free entries in the map.
558 * @param cls unused (NULL)
560 * @param entry entry of type "struct DownloadRequest" which is freed
564 free_entry (void *cls,
565 const GNUNET_HashCode *key,
574 * Stop a download (aborts if download is incomplete).
576 * @param dc handle for the download
577 * @param do_delete delete files of incomplete downloads
580 GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
583 // FIXME: make unpersistent
584 // FIXME: signal download end
586 if (GNUNET_SCHEDULER_NO_TASK != dc->task)
587 GNUNET_SCHEDULER_cancel (dc->h->sched,
589 if (NULL != dc->client)
590 GNUNET_CLIENT_disconnect (dc->client);
591 GNUNET_CONTAINER_multihashmap_iterate (dc->active,
594 GNUNET_CONTAINER_multihashmap_destroy (dc->active);
595 if (dc->filename != NULL)
597 GNUNET_DISK_file_close (dc->handle);
598 if ( (dc->completed != dc->length) &&
599 (GNUNET_YES == do_delete) )
601 if (0 != UNLINK (dc->filename))
602 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
606 GNUNET_free (dc->filename);
608 GNUNET_FS_uri_destroy (dc->uri);
632 * Compute how many bytes of data are stored in
636 get_node_size (const struct Node *node)
640 unsigned long long rsize;
641 unsigned long long spos;
642 unsigned long long epos;
644 GNUNET_GE_ASSERT (node->ctx->ectx, node->offset < node->ctx->total);
645 if (node->level == 0)
647 ret = GNUNET_ECRS_DBLOCK_SIZE;
648 if (node->offset + (unsigned long long) ret > node->ctx->total)
649 ret = (unsigned int) (node->ctx->total - node->offset);
651 GNUNET_GE_LOG (node->ctx->rm->ectx,
652 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
653 "Node at offset %llu and level %d has size %u\n",
654 node->offset, node->level, ret);
658 rsize = GNUNET_ECRS_DBLOCK_SIZE;
659 for (i = 0; i < node->level - 1; i++)
660 rsize *= GNUNET_ECRS_CHK_PER_INODE;
661 spos = rsize * (node->offset / sizeof (GNUNET_EC_ContentHashKey));
662 epos = spos + rsize * GNUNET_ECRS_CHK_PER_INODE;
663 if (epos > node->ctx->total)
664 epos = node->ctx->total;
665 ret = (epos - spos) / rsize;
666 if (ret * rsize < epos - spos)
667 ret++; /* need to round up! */
669 GNUNET_GE_LOG (node->ctx->rm->ectx,
670 GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER,
671 "Node at offset %llu and level %d has size %u\n",
672 node->offset, node->level,
673 ret * sizeof (GNUNET_EC_ContentHashKey));
675 return ret * sizeof (GNUNET_EC_ContentHashKey);
679 * Check if self block is already present on the drive. If the block
680 * is a dblock and present, the ProgressModel is notified. If the
681 * block is present and it is an iblock, downloading the children is
684 * Also checks if the block is within the range of blocks
685 * that we are supposed to download. If not, the method
686 * returns as if the block is present but does NOT signal
689 * @param node that is checked for presence
690 * @return GNUNET_YES if present, GNUNET_NO if not.
693 check_node_present (const struct Node *node)
701 size = get_node_size (node);
702 /* first check if node is within range.
703 For now, keeping it simple, we only do
704 this for level-0 nodes */
705 if ((node->level == 0) &&
706 ((node->offset + size < node->ctx->offset) ||
707 (node->offset >= node->ctx->offset + node->ctx->length)))
709 data = GNUNET_malloc (size);
711 res = read_from_files (node->ctx, node->level, node->offset, data, size);
714 GNUNET_hash (data, size, &hc);
715 if (0 == memcmp (&hc, &node->chk.key, sizeof (GNUNET_HashCode)))
717 notify_client_about_progress (node, data, size);
719 iblock_download_children (node, data, size);
730 /* end of fs_download.c */