src/fs/fs_download.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 2, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file fs/fs_download.c
  22  * @brief download methods
  23  * @author Christian Grothoff
  24  *
  25  * TODO:
  26  * - location URI suppport (can wait, easy)
  27  * - different priority for scheduling probe downloads?
  28  * - check if iblocks can be computed from existing blocks (can wait, hard)
  29  */
  30 #include "platform.h"
  31 #include "gnunet_constants.h"
  32 #include "gnunet_fs_service.h"
  33 #include "fs.h"
  34 #include "fs_tree.h"
  35
  36 #define DEBUG_DOWNLOAD GNUNET_NO
  37
  38 /**
  39  * Determine if the given download (options and meta data) should cause
  40  * use to try to do a recursive download.
  41  */
  42 static int
  43 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
  44 {
  45   return  (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
  46     ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
  47       ( (dc->meta == NULL) &&
  48         ( (NULL == dc->filename) ||
  49           ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
  50             (NULL !=
  51              strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
  52                      GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
  53 }
  54
  55
  56 /**
  57  * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
  58  * only have to truncate the file once we're done).
  59  *
  60  * Given the offset of a block (with respect to the DBLOCKS) and its
  61  * depth, return the offset where we would store this block in the
  62  * file.
  63  *
  64  * @param fsize overall file size
  65  * @param off offset of the block in the file
  66  * @param depth depth of the block in the tree
  67  * @param treedepth maximum depth of the tree
  68  * @return off for DBLOCKS (depth == treedepth),
  69  *         otherwise an offset past the end
  70  *         of the file that does not overlap
  71  *         with the range for any other block
  72  */
  73 static uint64_t
  74 compute_disk_offset (uint64_t fsize,
  75                      uint64_t off,
  76                      unsigned int depth,
  77                      unsigned int treedepth)
  78 {
  79   unsigned int i;
  80   uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
  81   uint64_t loff; /* where do IBlocks for depth "i" start? */
  82   unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
  83
  84   if (depth == treedepth)
  85     return off;
  86   /* first IBlocks start at the end of file, rounded up
  87      to full DBLOCK_SIZE */
  88   loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
  89   lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
  90   GNUNET_assert (0 == (off % DBLOCK_SIZE));
  91   ioff = (off / DBLOCK_SIZE);
  92   for (i=treedepth-1;i>depth;i--)
  93     {
  94       loff += lsize;
  95       lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
  96       GNUNET_assert (lsize > 0);
  97       GNUNET_assert (0 == (ioff % CHK_PER_INODE));
  98       ioff /= CHK_PER_INODE;
  99     }
 100   return loff + ioff * sizeof (struct ContentHashKey);
 101 }
 102
 103
 104 /**
 105  * Given a file of the specified treedepth and a block at the given
 106  * offset and depth, calculate the offset for the CHK at the given
 107  * index.
 108  *
 109  * @param offset the offset of the first
 110  *        DBLOCK in the subtree of the
 111  *        identified IBLOCK
 112  * @param depth the depth of the IBLOCK in the tree
 113  * @param treedepth overall depth of the tree
 114  * @param k which CHK in the IBLOCK are we
 115  *        talking about
 116  * @return offset if k=0, otherwise an appropriately
 117  *         larger value (i.e., if depth = treedepth-1,
 118  *         the returned value should be offset+DBLOCK_SIZE)
 119  */
 120 static uint64_t
 121 compute_dblock_offset (uint64_t offset,
 122                        unsigned int depth,
 123                        unsigned int treedepth,
 124                        unsigned int k)
 125 {
 126   unsigned int i;
 127   uint64_t lsize; /* what is the size of the sum of all DBlocks
 128                      that a CHK at depth i corresponds to? */
 129
 130   if (depth == treedepth)
 131     return offset;
 132   lsize = DBLOCK_SIZE;
 133   for (i=treedepth-1;i>depth;i--)
 134     lsize *= CHK_PER_INODE;
 135   return offset + k * lsize;
 136 }
 137
 138
 139 /**
 140  * Fill in all of the generic fields for a download event and call the
 141  * callback.
 142  *
 143  * @param pi structure to fill in
 144  * @param dc overall download context
 145  */
 146 void
 147 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
 148                                  struct GNUNET_FS_DownloadContext *dc)
 149 {
 150   pi->value.download.dc = dc;
 151   pi->value.download.cctx
 152     = dc->client_info;
 153   pi->value.download.pctx
 154     = (dc->parent == NULL) ? NULL : dc->parent->client_info;
 155   pi->value.download.sctx
 156     = (dc->search == NULL) ? NULL : dc->search->client_info;
 157   pi->value.download.uri
 158     = dc->uri;
 159   pi->value.download.filename
 160     = dc->filename;
 161   pi->value.download.size
 162     = dc->length;
 163   pi->value.download.duration
 164     = GNUNET_TIME_absolute_get_duration (dc->start_time);
 165   pi->value.download.completed
 166     = dc->completed;
 167   pi->value.download.anonymity
 168     = dc->anonymity;
 169   pi->value.download.eta
 170     = GNUNET_TIME_calculate_eta (dc->start_time,
 171                                  dc->completed,
 172                                  dc->length);
 173   if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
 174     dc->client_info = dc->h->upcb (dc->h->upcb_cls,
 175                                    pi);
 176   else
 177     dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
 178                                                         pi);
 179 }
 180
 181 /**
 182  * We're ready to transmit a search request to the
 183  * file-sharing service.  Do it.  If there is
 184  * more than one request pending, try to send
 185  * multiple or request another transmission.
 186  *
 187  * @param cls closure
 188  * @param size number of bytes available in buf
 189  * @param buf where the callee should write the message
 190  * @return number of bytes written to buf
 191  */
 192 static size_t
 193 transmit_download_request (void *cls,
 194                            size_t size,
 195                            void *buf);
 196
 197
 198 /**
 199  * Closure for iterator processing results.
 200  */
 201 struct ProcessResultClosure
 202 {
 203
 204   /**
 205    * Hash of data.
 206    */
 207   GNUNET_HashCode query;
 208
 209   /**
 210    * Data found in P2P network.
 211    */
 212   const void *data;
 213
 214   /**
 215    * Our download context.
 216    */
 217   struct GNUNET_FS_DownloadContext *dc;
 218
 219   /**
 220    * Number of bytes in data.
 221    */
 222   size_t size;
 223
 224   /**
 225    * Type of data.
 226    */
 227   enum GNUNET_BLOCK_Type type;
 228
 229   /**
 230    * Flag to indicate if this block should be stored on disk.
 231    */
 232   int do_store;
 233
 234 };
 235
 236
 237 /**
 238  * Iterator over entries in the pending requests in the 'active' map for the
 239  * reply that we just got.
 240  *
 241  * @param cls closure (our 'struct ProcessResultClosure')
 242  * @param key query for the given value / request
 243  * @param value value in the hash map (a 'struct DownloadRequest')
 244  * @return GNUNET_YES (we should continue to iterate); unless serious error
 245  */
 246 static int
 247 process_result_with_request (void *cls,
 248                              const GNUNET_HashCode * key,
 249                              void *value);
 250
 251
 252
 253 /**
 254  * Schedule the download of the specified block in the tree.
 255  *
 256  * @param dc overall download this block belongs to
 257  * @param chk content-hash-key of the block
 258  * @param offset offset of the block in the file
 259  *         (for IBlocks, the offset is the lowest
 260  *          offset of any DBlock in the subtree under
 261  *          the IBlock)
 262  * @param depth depth of the block, 0 is the root of the tree
 263  */
 264 static void
 265 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
 266                          const struct ContentHashKey *chk,
 267                          uint64_t offset,
 268                          unsigned int depth)
 269 {
 270   struct DownloadRequest *sm;
 271   uint64_t total;
 272   uint64_t off;
 273   size_t len;
 274   char block[DBLOCK_SIZE];
 275   GNUNET_HashCode key;
 276   struct ProcessResultClosure prc;
 277   struct GNUNET_DISK_FileHandle *fh;
 278
 279 #if DEBUG_DOWNLOAD
 280   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 281               "Scheduling download at offset %llu and depth %u for `%s'\n",
 282               (unsigned long long) offset,
 283               depth,
 284               GNUNET_h2s (&chk->query));
 285 #endif
 286   total = GNUNET_ntohll (dc->uri->data.chk.file_length);
 287   off = compute_disk_offset (total,
 288                              offset,
 289                              depth,
 290                              dc->treedepth);
 291   len = GNUNET_FS_tree_calculate_block_size (total,
 292                                              dc->treedepth,
 293                                              offset,
 294                                              depth);
 295   sm = GNUNET_malloc (sizeof (struct DownloadRequest));
 296   sm->chk = *chk;
 297   sm->offset = offset;
 298   sm->depth = depth;
 299   sm->is_pending = GNUNET_YES;
 300   sm->next = dc->pending;
 301   dc->pending = sm;
 302   GNUNET_CONTAINER_multihashmap_put (dc->active,
 303                                      &chk->query,
 304                                      sm,
 305                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
 306   fh = NULL;
 307   if ( (dc->old_file_size > off) &&
 308        (dc->filename != NULL) )
 309     fh = GNUNET_DISK_file_open (dc->filename,
 310                                 GNUNET_DISK_OPEN_READ,
 311                                 GNUNET_DISK_PERM_NONE);
 312   if ( (fh != NULL) &&
 313        (off  ==
 314         GNUNET_DISK_file_seek (fh,
 315                                off,
 316                                GNUNET_DISK_SEEK_SET) ) &&
 317        (len ==
 318         GNUNET_DISK_file_read (fh,
 319                                block,
 320                                len)) )
 321     {
 322       GNUNET_CRYPTO_hash (block, len, &key);
 323       if (0 == memcmp (&key,
 324                        &chk->key,
 325                        sizeof (GNUNET_HashCode)))
 326         {
 327           char enc[len];
 328           struct GNUNET_CRYPTO_AesSessionKey sk;
 329           struct GNUNET_CRYPTO_AesInitializationVector iv;
 330           GNUNET_HashCode query;
 331
 332           GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
 333           GNUNET_CRYPTO_aes_encrypt (block, len,
 334                                      &sk,
 335                                      &iv,
 336                                      enc);
 337           GNUNET_CRYPTO_hash (enc, len, &query);
 338           if (0 == memcmp (&query,
 339                            &chk->query,
 340                            sizeof (GNUNET_HashCode)))
 341             {
 342 #if DEBUG_DOWNLOAD
 343               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 344                           "Matching block already present, no need for download!\n");
 345 #endif
 346               /* already got it! */
 347               prc.dc = dc;
 348               prc.data = enc;
 349               prc.size = len;
 350               prc.type = (dc->treedepth == depth)
 351                 ? GNUNET_BLOCK_TYPE_DBLOCK
 352                 : GNUNET_BLOCK_TYPE_IBLOCK;
 353               prc.query = chk->query;
 354               prc.do_store = GNUNET_NO; /* useless */
 355               process_result_with_request (&prc,
 356                                            &key,
 357                                            sm);
 358             }
 359           else
 360             {
 361               GNUNET_break_op (0);
 362             }
 363           GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 364           return;
 365         }
 366     }
 367   if (fh != NULL)
 368     GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 369   if (depth < dc->treedepth)
 370     {
 371       // FIXME: try if we could
 372       // reconstitute this IBLOCK
 373       // from the existing blocks on disk (can wait)
 374       // (read block(s), encode, compare with
 375       // query; if matches, simply return)
 376     }
 377
 378   if ( (dc->th == NULL) &&
 379        (dc->client != NULL) )
 380     {
 381 #if DEBUG_DOWNLOAD
 382       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 383                   "Asking for transmission to FS service\n");
 384 #endif
 385       dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
 386                                                     sizeof (struct SearchMessage),
 387                                                     GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 388                                                     GNUNET_NO,
 389                                                     &transmit_download_request,
 390                                                     dc);
 391     }
 392 }
 393
 394
 395
 396 /**
 397  * Suggest a filename based on given metadata.
 398  *
 399  * @param md given meta data
 400  * @return NULL if meta data is useless for suggesting a filename
 401  */
 402 char *
 403 GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md)
 404 {
 405   static const char *mimeMap[][2] = {
 406     {"application/bz2", ".bz2"},
 407     {"application/gnunet-directory", ".gnd"},
 408     {"application/java", ".class"},
 409     {"application/msword", ".doc"},
 410     {"application/ogg", ".ogg"},
 411     {"application/pdf", ".pdf"},
 412     {"application/pgp-keys", ".key"},
 413     {"application/pgp-signature", ".pgp"},
 414     {"application/postscript", ".ps"},
 415     {"application/rar", ".rar"},
 416     {"application/rtf", ".rtf"},
 417     {"application/xml", ".xml"},
 418     {"application/x-debian-package", ".deb"},
 419     {"application/x-dvi", ".dvi"},
 420     {"applixation/x-flac", ".flac"},
 421     {"applixation/x-gzip", ".gz"},
 422     {"application/x-java-archive", ".jar"},
 423     {"application/x-java-vm", ".class"},
 424     {"application/x-python-code", ".pyc"},
 425     {"application/x-redhat-package-manager", ".rpm"},
 426     {"application/x-rpm", ".rpm"},
 427     {"application/x-tar", ".tar"},
 428     {"application/x-tex-pk", ".pk"},
 429     {"application/x-texinfo", ".texinfo"},
 430     {"application/x-xcf", ".xcf"},
 431     {"application/x-xfig", ".xfig"},
 432     {"application/zip", ".zip"},
 433
 434     {"audio/midi", ".midi"},
 435     {"audio/mpeg", ".mp3"},
 436     {"audio/real", ".rm"},
 437     {"audio/x-wav", ".wav"},
 438
 439     {"image/gif", ".gif"},
 440     {"image/jpeg", ".jpg"},
 441     {"image/pcx", ".pcx"},
 442     {"image/png", ".png"},
 443     {"image/tiff", ".tiff"},
 444     {"image/x-ms-bmp", ".bmp"},
 445     {"image/x-xpixmap", ".xpm"},
 446
 447     {"text/css", ".css"},
 448     {"text/html", ".html"},
 449     {"text/plain", ".txt"},
 450     {"text/rtf", ".rtf"},
 451     {"text/x-c++hdr", ".h++"},
 452     {"text/x-c++src", ".c++"},
 453     {"text/x-chdr", ".h"},
 454     {"text/x-csrc", ".c"},
 455     {"text/x-java", ".java"},
 456     {"text/x-moc", ".moc"},
 457     {"text/x-pascal", ".pas"},
 458     {"text/x-perl", ".pl"},
 459     {"text/x-python", ".py"},
 460     {"text/x-tex", ".tex"},
 461
 462     {"video/avi", ".avi"},
 463     {"video/mpeg", ".mpeg"},
 464     {"video/quicktime", ".qt"},
 465     {"video/real", ".rm"},
 466     {"video/x-msvideo", ".avi"},
 467     {NULL, NULL},
 468   };
 469   char *ret;
 470   unsigned int i;
 471   char *mime;
 472   char *base;
 473   const char *ext;
 474
 475   ret = GNUNET_CONTAINER_meta_data_get_by_type (md,
 476                                                 EXTRACTOR_METATYPE_FILENAME);
 477   if (ret != NULL)
 478     return ret;
 479   ext = NULL;
 480   mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
 481                                                  EXTRACTOR_METATYPE_MIMETYPE);
 482   if (mime != NULL)
 483     {
 484       i = 0;
 485       while ( (mimeMap[i][0] != NULL) &&
 486               (0 != strcmp (mime, mimeMap[i][0])))
 487         i++;
 488       if (mimeMap[i][1] == NULL)
 489         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG |
 490                     GNUNET_ERROR_TYPE_BULK,
 491                     _("Did not find mime type `%s' in extension list.\n"),
 492                     mime);
 493       else
 494         ext = mimeMap[i][1];
 495       GNUNET_free (mime);
 496     }
 497   base = GNUNET_CONTAINER_meta_data_get_first_by_types (md,
 498                                                         EXTRACTOR_METATYPE_TITLE,
 499                                                         EXTRACTOR_METATYPE_BOOK_TITLE,
 500                                                         EXTRACTOR_METATYPE_ORIGINAL_TITLE,
 501                                                         EXTRACTOR_METATYPE_PACKAGE_NAME,
 502                                                         EXTRACTOR_METATYPE_URL,
 503                                                         EXTRACTOR_METATYPE_URI,
 504                                                         EXTRACTOR_METATYPE_DESCRIPTION,
 505                                                         EXTRACTOR_METATYPE_ISRC,
 506                                                         EXTRACTOR_METATYPE_JOURNAL_NAME,
 507                                                         EXTRACTOR_METATYPE_AUTHOR_NAME,
 508                                                         EXTRACTOR_METATYPE_SUBJECT,
 509                                                         EXTRACTOR_METATYPE_ALBUM,
 510                                                         EXTRACTOR_METATYPE_ARTIST,
 511                                                         EXTRACTOR_METATYPE_KEYWORDS,
 512                                                         EXTRACTOR_METATYPE_COMMENT,
 513                                                         EXTRACTOR_METATYPE_UNKNOWN,
 514                                                         -1);
 515   if ( (base == NULL) &&
 516        (ext == NULL) )
 517     return NULL;
 518   if (base == NULL)
 519     return GNUNET_strdup (ext);
 520   if (ext == NULL)
 521     return base;
 522   GNUNET_asprintf (&ret,
 523                    "%s%s",
 524                    base,
 525                    ext);
 526   GNUNET_free (base);
 527   return ret;
 528 }
 529
 530
 531 /**
 532  * We've lost our connection with the FS service.
 533  * Re-establish it and re-transmit all of our
 534  * pending requests.
 535  *
 536  * @param dc download context that is having trouble
 537  */
 538 static void
 539 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 540
 541
 542 /**
 543  * We found an entry in a directory.  Check if the respective child
 544  * already exists and if not create the respective child download.
 545  *
 546  * @param cls the parent download
 547  * @param filename name of the file in the directory
 548  * @param uri URI of the file (CHK or LOC)
 549  * @param meta meta data of the file
 550  * @param length number of bytes in data
 551  * @param data contents of the file (or NULL if they were not inlined)
 552  */
 553 static void
 554 trigger_recursive_download (void *cls,
 555                             const char *filename,
 556                             const struct GNUNET_FS_Uri *uri,
 557                             const struct GNUNET_CONTAINER_MetaData *meta,
 558                             size_t length,
 559                             const void *data);
 560
 561
 562 /**
 563  * We're done downloading a directory.  Open the file and
 564  * trigger all of the (remaining) child downloads.
 565  *
 566  * @param dc context of download that just completed
 567  */
 568 static void
 569 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 570 {
 571   size_t size;
 572   uint64_t size64;
 573   void *data;
 574   struct GNUNET_DISK_FileHandle *h;
 575   struct GNUNET_DISK_MapHandle *m;
 576
 577   size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
 578   size = (size_t) size64;
 579   if (size64 != (uint64_t) size)
 580     {
 581       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 582                   _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
 583       return;
 584     }
 585   if (dc->filename != NULL)
 586     {
 587       h = GNUNET_DISK_file_open (dc->filename,
 588                                  GNUNET_DISK_OPEN_READ,
 589                                  GNUNET_DISK_PERM_NONE);
 590     }
 591   else
 592     {
 593       GNUNET_assert (dc->temp_filename != NULL);
 594       h = GNUNET_DISK_file_open (dc->temp_filename,
 595                                  GNUNET_DISK_OPEN_READ,
 596                                  GNUNET_DISK_PERM_NONE);
 597     }
 598   if (h == NULL)
 599     return; /* oops */
 600   data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
 601   if (data == NULL)
 602     {
 603       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 604                   _("Directory too large for system address space\n"));
 605     }
 606   else
 607     {
 608       GNUNET_FS_directory_list_contents (size,
 609                                          data,
 610                                          0,
 611                                          &trigger_recursive_download,
 612                                          dc);
 613       GNUNET_DISK_file_unmap (m);
 614     }
 615   GNUNET_DISK_file_close (h);
 616   if (dc->filename == NULL)
 617     {
 618       if (0 != UNLINK (dc->temp_filename))
 619         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
 620                                   "unlink",
 621                                   dc->temp_filename);
 622       GNUNET_free (dc->temp_filename);
 623       dc->temp_filename = NULL;
 624     }
 625 }
 626
 627
 628 /**
 629  * Check if all child-downloads have completed and
 630  * if so, signal completion (and possibly recurse to
 631  * parent).
 632  */
 633 static void
 634 check_completed (struct GNUNET_FS_DownloadContext *dc)
 635 {
 636   struct GNUNET_FS_ProgressInfo pi;
 637   struct GNUNET_FS_DownloadContext *pos;
 638
 639   pos = dc->child_head;
 640   while (pos != NULL)
 641     {
 642       if ( (pos->emsg == NULL) &&
 643            (pos->completed < pos->length) )
 644         return; /* not done yet */
 645       if ( (pos->child_head != NULL) &&
 646            (pos->has_finished != GNUNET_YES) )
 647         return; /* not transitively done yet */
 648       pos = pos->next;
 649     }
 650   dc->has_finished = GNUNET_YES;
 651   GNUNET_FS_download_sync_ (dc);
 652   /* signal completion */
 653   pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
 654   GNUNET_FS_download_make_status_ (&pi, dc);
 655   if (dc->parent != NULL)
 656     check_completed (dc->parent);
 657 }
 658
 659
 660 /**
 661  * We found an entry in a directory.  Check if the respective child
 662  * already exists and if not create the respective child download.
 663  *
 664  * @param cls the parent download
 665  * @param filename name of the file in the directory
 666  * @param uri URI of the file (CHK or LOC)
 667  * @param meta meta data of the file
 668  * @param length number of bytes in data
 669  * @param data contents of the file (or NULL if they were not inlined)
 670  */
 671 static void
 672 trigger_recursive_download (void *cls,
 673                             const char *filename,
 674                             const struct GNUNET_FS_Uri *uri,
 675                             const struct GNUNET_CONTAINER_MetaData *meta,
 676                             size_t length,
 677                             const void *data)
 678 {
 679   struct GNUNET_FS_DownloadContext *dc = cls;
 680   struct GNUNET_FS_DownloadContext *cpos;
 681   struct GNUNET_DISK_FileHandle *fh;
 682   char *temp_name;
 683   const char *real_name;
 684   char *fn;
 685   char *us;
 686   char *ext;
 687   char *dn;
 688   char *full_name;
 689
 690   if (NULL == uri)
 691     return; /* entry for the directory itself */
 692   cpos = dc->child_head;
 693   while (cpos != NULL)
 694     {
 695       if ( (GNUNET_FS_uri_test_equal (uri,
 696                                       cpos->uri)) ||
 697            ( (filename != NULL) &&
 698              (0 == strcmp (cpos->filename,
 699                            filename)) ) )
 700         break;
 701       cpos = cpos->next;
 702     }
 703   if (cpos != NULL)
 704     return; /* already exists */
 705   fn = NULL;
 706   if (NULL == filename)
 707     {
 708       fn = GNUNET_FS_meta_data_suggest_filename (meta);
 709       if (fn == NULL)
 710         {
 711           us = GNUNET_FS_uri_to_string (uri);
 712           fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_PREFIX
 713                                            GNUNET_FS_URI_CHK_INFIX)]);
 714           GNUNET_free (us);
 715         }
 716       else if (fn[0] == '.')
 717         {
 718           ext = fn;
 719           us = GNUNET_FS_uri_to_string (uri);
 720           GNUNET_asprintf (&fn,
 721                            "%s%s",
 722                            &us[strlen (GNUNET_FS_URI_PREFIX
 723                                        GNUNET_FS_URI_CHK_INFIX)], ext);
 724           GNUNET_free (ext);
 725           GNUNET_free (us);
 726         }
 727       filename = fn;
 728     }
 729   if (dc->filename == NULL)
 730     {
 731       full_name = NULL;
 732     }
 733   else
 734     {
 735       dn = GNUNET_strdup (dc->filename);
 736       GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 737                      (NULL !=
 738                       strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 739                               GNUNET_FS_DIRECTORY_EXT)) );
 740       if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 741            (NULL !=
 742             strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 743                     GNUNET_FS_DIRECTORY_EXT)) )
 744         dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
 745       if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
 746            ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
 747              (NULL ==
 748               strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
 749                       GNUNET_FS_DIRECTORY_EXT)) ) )
 750         {
 751           GNUNET_asprintf (&full_name,
 752                            "%s%s%s%s",
 753                            dn,
 754                            DIR_SEPARATOR_STR,
 755                            filename,
 756                            GNUNET_FS_DIRECTORY_EXT);
 757         }
 758       else
 759         {
 760           GNUNET_asprintf (&full_name,
 761                            "%s%s%s",
 762                            dn,
 763                            DIR_SEPARATOR_STR,
 764                            filename);
 765         }
 766       GNUNET_free (dn);
 767     }
 768   if ( (full_name != NULL) &&
 769        (GNUNET_OK !=
 770         GNUNET_DISK_directory_create_for_file (full_name)) )
 771     {
 772       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 773                   _("Failed to create directory for recursive download of `%s'\n"),
 774                   full_name);
 775       GNUNET_free (full_name);
 776       GNUNET_free_non_null (fn);
 777       return;
 778     }
 779
 780   temp_name = NULL;
 781   if ( (data != NULL) &&
 782        (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
 783     {
 784       if (full_name == NULL)
 785         {
 786           temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
 787           real_name = temp_name;
 788         }
 789       else
 790         {
 791           real_name = full_name;
 792         }
 793       /* write to disk, then trigger normal download which will instantly progress to completion */
 794       fh = GNUNET_DISK_file_open (real_name,
 795                                   GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
 796                                   GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
 797       if (fh == NULL)
 798         {
 799           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 800                                     "open",
 801                                     real_name);
 802           GNUNET_free (full_name);
 803           GNUNET_free_non_null (fn);
 804           return;
 805         }
 806       if (length !=
 807           GNUNET_DISK_file_write (fh,
 808                                   data,
 809                                   length))
 810         {
 811           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 812                                     "write",
 813                                     full_name);
 814         }
 815       GNUNET_DISK_file_close (fh);
 816     }
 817   GNUNET_FS_download_start (dc->h,
 818                             uri,
 819                             meta,
 820                             full_name, temp_name,
 821                             0,
 822                             GNUNET_FS_uri_chk_get_file_size (uri),
 823                             dc->anonymity,
 824                             dc->options,
 825                             NULL,
 826                             dc);
 827   GNUNET_free_non_null (full_name);
 828   GNUNET_free_non_null (temp_name);
 829   GNUNET_free_non_null (fn);
 830 }
 831
 832
 833 /**
 834  * Free entries in the map.
 835  *
 836  * @param cls unused (NULL)
 837  * @param key unused
 838  * @param entry entry of type "struct DownloadRequest" which is freed
 839  * @return GNUNET_OK
 840  */
 841 static int
 842 free_entry (void *cls,
 843             const GNUNET_HashCode *key,
 844             void *entry)
 845 {
 846   GNUNET_free (entry);
 847   return GNUNET_OK;
 848 }
 849
 850
 851 /**
 852  * Iterator over entries in the pending requests in the 'active' map for the
 853  * reply that we just got.
 854  *
 855  * @param cls closure (our 'struct ProcessResultClosure')
 856  * @param key query for the given value / request
 857  * @param value value in the hash map (a 'struct DownloadRequest')
 858  * @return GNUNET_YES (we should continue to iterate); unless serious error
 859  */
 860 static int
 861 process_result_with_request (void *cls,
 862                              const GNUNET_HashCode * key,
 863                              void *value)
 864 {
 865   struct ProcessResultClosure *prc = cls;
 866   struct DownloadRequest *sm = value;
 867   struct DownloadRequest *ppos;
 868   struct DownloadRequest *pprev;
 869   struct GNUNET_DISK_FileHandle *fh;
 870   struct GNUNET_FS_DownloadContext *dc = prc->dc;
 871   struct GNUNET_CRYPTO_AesSessionKey skey;
 872   struct GNUNET_CRYPTO_AesInitializationVector iv;
 873   char pt[prc->size];
 874   struct GNUNET_FS_ProgressInfo pi;
 875   uint64_t off;
 876   size_t bs;
 877   size_t app;
 878   int i;
 879   struct ContentHashKey *chk;
 880
 881   fh = NULL;
 882   bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
 883                                             dc->treedepth,
 884                                             sm->offset,
 885                                             sm->depth);
 886   if (prc->size != bs)
 887     {
 888 #if DEBUG_DOWNLOAD
 889       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 890                   "Internal error or bogus download URI (expected %u bytes, got %u)\n",
 891                   bs,
 892                   prc->size);
 893 #endif
 894       dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
 895       goto signal_error;
 896     }
 897   GNUNET_assert (GNUNET_YES ==
 898                  GNUNET_CONTAINER_multihashmap_remove (dc->active,
 899                                                        &prc->query,
 900                                                        sm));
 901   /* if this request is on the pending list, remove it! */
 902   pprev = NULL;
 903   ppos = dc->pending;
 904   while (ppos != NULL)
 905     {
 906       if (ppos == sm)
 907         {
 908           if (pprev == NULL)
 909             dc->pending = ppos->next;
 910           else
 911             pprev->next = ppos->next;
 912           break;
 913         }
 914       pprev = ppos;
 915       ppos = ppos->next;
 916     }
 917   GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
 918   GNUNET_CRYPTO_aes_decrypt (prc->data,
 919                              prc->size,
 920                              &skey,
 921                              &iv,
 922                              pt);
 923   off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
 924                              sm->offset,
 925                              sm->depth,
 926                              dc->treedepth);
 927   /* save to disk */
 928   if ( ( GNUNET_YES == prc->do_store) &&
 929        ( (dc->filename != NULL) ||
 930          (is_recursive_download (dc)) ) &&
 931        ( (sm->depth == dc->treedepth) ||
 932          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 933     {
 934       fh = GNUNET_DISK_file_open (dc->filename != NULL
 935                                   ? dc->filename
 936                                   : dc->temp_filename,
 937                                   GNUNET_DISK_OPEN_READWRITE |
 938                                   GNUNET_DISK_OPEN_CREATE,
 939                                   GNUNET_DISK_PERM_USER_READ |
 940                                   GNUNET_DISK_PERM_USER_WRITE |
 941                                   GNUNET_DISK_PERM_GROUP_READ |
 942                                   GNUNET_DISK_PERM_OTHER_READ);
 943     }
 944   if ( (NULL == fh) &&
 945        (GNUNET_YES == prc->do_store) &&
 946        ( (dc->filename != NULL) ||
 947          (is_recursive_download (dc)) ) &&
 948        ( (sm->depth == dc->treedepth) ||
 949          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 950     {
 951       GNUNET_asprintf (&dc->emsg,
 952                        _("Download failed: could not open file `%s': %s\n"),
 953                        dc->filename,
 954                        STRERROR (errno));
 955       goto signal_error;
 956     }
 957   if (fh != NULL)
 958     {
 959 #if DEBUG_DOWNLOAD
 960       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 961                   "Saving decrypted block to disk at offset %llu\n",
 962                   (unsigned long long) off);
 963 #endif
 964       if ( (off  !=
 965             GNUNET_DISK_file_seek (fh,
 966                                    off,
 967                                    GNUNET_DISK_SEEK_SET) ) )
 968         {
 969           GNUNET_asprintf (&dc->emsg,
 970                            _("Failed to seek to offset %llu in file `%s': %s\n"),
 971                            (unsigned long long) off,
 972                            dc->filename,
 973                            STRERROR (errno));
 974           goto signal_error;
 975         }
 976       if (prc->size !=
 977           GNUNET_DISK_file_write (fh,
 978                                   pt,
 979                                   prc->size))
 980         {
 981           GNUNET_asprintf (&dc->emsg,
 982                            _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
 983                            (unsigned int) prc->size,
 984                            (unsigned long long) off,
 985                            dc->filename,
 986                            STRERROR (errno));
 987           goto signal_error;
 988         }
 989       GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 990       fh = NULL;
 991     }
 992   if (sm->depth == dc->treedepth)
 993     {
 994       app = prc->size;
 995       if (sm->offset < dc->offset)
 996         {
 997           /* starting offset begins in the middle of pt,
 998              do not count first bytes as progress */
 999           GNUNET_assert (app > (dc->offset - sm->offset));
1000           app -= (dc->offset - sm->offset);
1001         }
1002       if (sm->offset + prc->size > dc->offset + dc->length)
1003         {
1004           /* end of block is after relevant range,
1005              do not count last bytes as progress */
1006           GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length));
1007           app -= (sm->offset + prc->size) - (dc->offset + dc->length);
1008         }
1009       dc->completed += app;
1010
1011       /* do recursive download if option is set and either meta data
1012          says it is a directory or if no meta data is given AND filename
1013          ends in '.gnd' (top-level case) */
1014       if (is_recursive_download (dc))
1015         GNUNET_FS_directory_list_contents (prc->size,
1016                                            pt,
1017                                            off,
1018                                            &trigger_recursive_download,
1019                                            dc);
1020
1021     }
1022   pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1023   pi.value.download.specifics.progress.data = pt;
1024   pi.value.download.specifics.progress.offset = sm->offset;
1025   pi.value.download.specifics.progress.data_len = prc->size;
1026   pi.value.download.specifics.progress.depth = sm->depth;
1027   GNUNET_FS_download_make_status_ (&pi, dc);
1028   GNUNET_assert (dc->completed <= dc->length);
1029   if (dc->completed == dc->length)
1030     {
1031 #if DEBUG_DOWNLOAD
1032       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1033                   "Download completed, truncating file to desired length %llu\n",
1034                   (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1035 #endif
1036       /* truncate file to size (since we store IBlocks at the end) */
1037       if (dc->filename != NULL)
1038         {
1039           if (0 != truncate (dc->filename,
1040                              GNUNET_ntohll (dc->uri->data.chk.file_length)))
1041             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1042                                       "truncate",
1043                                       dc->filename);
1044         }
1045       if (dc->job_queue != NULL)
1046         {
1047           GNUNET_FS_dequeue_ (dc->job_queue);
1048           dc->job_queue = NULL;
1049         }
1050       if (is_recursive_download (dc))
1051         full_recursive_download (dc);
1052       if (dc->child_head == NULL)
1053         {
1054           /* signal completion */
1055           pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
1056           GNUNET_FS_download_make_status_ (&pi, dc);
1057           if (dc->parent != NULL)
1058             check_completed (dc->parent);
1059         }
1060       GNUNET_assert (sm->depth == dc->treedepth);
1061     }
1062   if (sm->depth == dc->treedepth)
1063     {
1064       GNUNET_FS_download_sync_ (dc);
1065       GNUNET_free (sm);
1066       return GNUNET_YES;
1067     }
1068 #if DEBUG_DOWNLOAD
1069   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1070               "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1071               sm->depth,
1072               (unsigned long long) sm->offset);
1073 #endif
1074   GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1075   chk = (struct ContentHashKey*) pt;
1076   for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1077     {
1078       off = compute_dblock_offset (sm->offset,
1079                                    sm->depth,
1080                                    dc->treedepth,
1081                                    i);
1082       if ( (off + DBLOCK_SIZE >= dc->offset) &&
1083            (off < dc->offset + dc->length) )
1084         schedule_block_download (dc,
1085                                  &chk[i],
1086                                  off,
1087                                  sm->depth + 1);
1088     }
1089   GNUNET_free (sm);
1090   GNUNET_FS_download_sync_ (dc);
1091   return GNUNET_YES;
1092
1093  signal_error:
1094   if (fh != NULL)
1095     GNUNET_DISK_file_close (fh);
1096   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1097   pi.value.download.specifics.error.message = dc->emsg;
1098   GNUNET_FS_download_make_status_ (&pi, dc);
1099   /* abort all pending requests */
1100   if (NULL != dc->th)
1101     {
1102       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1103       dc->th = NULL;
1104     }
1105   GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1106   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1107                                          &free_entry,
1108                                          NULL);
1109   dc->pending = NULL;
1110   dc->client = NULL;
1111   GNUNET_free (sm);
1112   GNUNET_FS_download_sync_ (dc);
1113   return GNUNET_NO;
1114 }
1115
1116
1117 /**
1118  * Process a download result.
1119  *
1120  * @param dc our download context
1121  * @param type type of the result
1122  * @param data the (encrypted) response
1123  * @param size size of data
1124  */
1125 static void
1126 process_result (struct GNUNET_FS_DownloadContext *dc,
1127                 enum GNUNET_BLOCK_Type type,
1128                 const void *data,
1129                 size_t size)
1130 {
1131   struct ProcessResultClosure prc;
1132
1133   prc.dc = dc;
1134   prc.data = data;
1135   prc.size = size;
1136   prc.type = type;
1137   prc.do_store = GNUNET_YES;
1138   GNUNET_CRYPTO_hash (data, size, &prc.query);
1139 #if DEBUG_DOWNLOAD
1140   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1141               "Received result for query `%s' from `%s'-service\n",
1142               GNUNET_h2s (&prc.query),
1143               "FS");
1144 #endif
1145   GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1146                                               &prc.query,
1147                                               &process_result_with_request,
1148                                               &prc);
1149 }
1150
1151
1152 /**
1153  * Type of a function to call when we receive a message
1154  * from the service.
1155  *
1156  * @param cls closure
1157  * @param msg message received, NULL on timeout or fatal error
1158  */
1159 static void
1160 receive_results (void *cls,
1161                  const struct GNUNET_MessageHeader * msg)
1162 {
1163   struct GNUNET_FS_DownloadContext *dc = cls;
1164   const struct PutMessage *cm;
1165   uint16_t msize;
1166
1167   if ( (NULL == msg) ||
1168        (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1169        (sizeof (struct PutMessage) > ntohs(msg->size)) )
1170     {
1171       GNUNET_break (msg == NULL);
1172       try_reconnect (dc);
1173       return;
1174     }
1175   msize = ntohs(msg->size);
1176   cm = (const struct PutMessage*) msg;
1177   process_result (dc,
1178                   ntohl (cm->type),
1179                   &cm[1],
1180                   msize - sizeof (struct PutMessage));
1181   if (dc->client == NULL)
1182     return; /* fatal error */
1183   /* continue receiving */
1184   GNUNET_CLIENT_receive (dc->client,
1185                          &receive_results,
1186                          dc,
1187                          GNUNET_TIME_UNIT_FOREVER_REL);
1188 }
1189
1190
1191
1192 /**
1193  * We're ready to transmit a search request to the
1194  * file-sharing service.  Do it.  If there is
1195  * more than one request pending, try to send
1196  * multiple or request another transmission.
1197  *
1198  * @param cls closure
1199  * @param size number of bytes available in buf
1200  * @param buf where the callee should write the message
1201  * @return number of bytes written to buf
1202  */
1203 static size_t
1204 transmit_download_request (void *cls,
1205                            size_t size,
1206                            void *buf)
1207 {
1208   struct GNUNET_FS_DownloadContext *dc = cls;
1209   size_t msize;
1210   struct SearchMessage *sm;
1211
1212   dc->th = NULL;
1213   if (NULL == buf)
1214     {
1215 #if DEBUG_DOWNLOAD
1216       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1217                   "Transmitting download request failed, trying to reconnect\n");
1218 #endif
1219       try_reconnect (dc);
1220       return 0;
1221     }
1222   GNUNET_assert (size >= sizeof (struct SearchMessage));
1223   msize = 0;
1224   sm = buf;
1225   while ( (dc->pending != NULL) &&
1226           (size > msize + sizeof (struct SearchMessage)) )
1227     {
1228 #if DEBUG_DOWNLOAD
1229       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1230                   "Transmitting download request for `%s' to `%s'-service\n",
1231                   GNUNET_h2s (&dc->pending->chk.query),
1232                   "FS");
1233 #endif
1234       memset (sm, 0, sizeof (struct SearchMessage));
1235       sm->header.size = htons (sizeof (struct SearchMessage));
1236       sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1237       if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1238         sm->options = htonl (1);
1239       else
1240         sm->options = htonl (0);
1241       if (dc->pending->depth == dc->treedepth)
1242         sm->type = htonl (GNUNET_BLOCK_TYPE_DBLOCK);
1243       else
1244         sm->type = htonl (GNUNET_BLOCK_TYPE_IBLOCK);
1245       sm->anonymity_level = htonl (dc->anonymity);
1246       sm->target = dc->target.hashPubKey;
1247       sm->query = dc->pending->chk.query;
1248       dc->pending->is_pending = GNUNET_NO;
1249       dc->pending = dc->pending->next;
1250       msize += sizeof (struct SearchMessage);
1251       sm++;
1252     }
1253   if (dc->pending != NULL)
1254     dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1255                                                   sizeof (struct SearchMessage),
1256                                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1257                                                   GNUNET_NO,
1258                                                   &transmit_download_request,
1259                                                   dc);
1260   return msize;
1261 }
1262
1263
1264 /**
1265  * Reconnect to the FS service and transmit our queries NOW.
1266  *
1267  * @param cls our download context
1268  * @param tc unused
1269  */
1270 static void
1271 do_reconnect (void *cls,
1272               const struct GNUNET_SCHEDULER_TaskContext *tc)
1273 {
1274   struct GNUNET_FS_DownloadContext *dc = cls;
1275   struct GNUNET_CLIENT_Connection *client;
1276
1277   dc->task = GNUNET_SCHEDULER_NO_TASK;
1278   client = GNUNET_CLIENT_connect (dc->h->sched,
1279                                   "fs",
1280                                   dc->h->cfg);
1281   if (NULL == client)
1282     {
1283       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1284                   "Connecting to `%s'-service failed, will try again.\n",
1285                   "FS");
1286       try_reconnect (dc);
1287       return;
1288     }
1289   dc->client = client;
1290   dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1291                                                 sizeof (struct SearchMessage),
1292                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1293                                                 GNUNET_NO,
1294                                                 &transmit_download_request,
1295                                                 dc);
1296   GNUNET_CLIENT_receive (client,
1297                          &receive_results,
1298                          dc,
1299                          GNUNET_TIME_UNIT_FOREVER_REL);
1300 }
1301
1302
1303 /**
1304  * Add entries that are not yet pending back to the pending list.
1305  *
1306  * @param cls our download context
1307  * @param key unused
1308  * @param entry entry of type "struct DownloadRequest"
1309  * @return GNUNET_OK
1310  */
1311 static int
1312 retry_entry (void *cls,
1313              const GNUNET_HashCode *key,
1314              void *entry)
1315 {
1316   struct GNUNET_FS_DownloadContext *dc = cls;
1317   struct DownloadRequest *dr = entry;
1318
1319   if (! dr->is_pending)
1320     {
1321       dr->next = dc->pending;
1322       dr->is_pending = GNUNET_YES;
1323       dc->pending = entry;
1324     }
1325   return GNUNET_OK;
1326 }
1327
1328
1329 /**
1330  * We've lost our connection with the FS service.
1331  * Re-establish it and re-transmit all of our
1332  * pending requests.
1333  *
1334  * @param dc download context that is having trouble
1335  */
1336 static void
1337 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1338 {
1339
1340   if (NULL != dc->client)
1341     {
1342 #if DEBUG_DOWNLOAD
1343       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1344                   "Moving all requests back to pending list\n");
1345 #endif
1346       if (NULL != dc->th)
1347         {
1348           GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1349           dc->th = NULL;
1350         }
1351       GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1352                                              &retry_entry,
1353                                              dc);
1354       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1355       dc->client = NULL;
1356     }
1357 #if DEBUG_DOWNLOAD
1358   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1359               "Will try to reconnect in 1s\n");
1360 #endif
1361   dc->task
1362     = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
1363                                     GNUNET_TIME_UNIT_SECONDS,
1364                                     &do_reconnect,
1365                                     dc);
1366 }
1367
1368
1369
1370 /**
1371  * We're allowed to ask the FS service for our blocks.  Start the download.
1372  *
1373  * @param cls the 'struct GNUNET_FS_DownloadContext'
1374  * @param client handle to use for communcation with FS (we must destroy it!)
1375  */
1376 static void
1377 activate_fs_download (void *cls,
1378                       struct GNUNET_CLIENT_Connection *client)
1379 {
1380   struct GNUNET_FS_DownloadContext *dc = cls;
1381   struct GNUNET_FS_ProgressInfo pi;
1382
1383   GNUNET_assert (NULL != client);
1384   GNUNET_assert (dc->client == NULL);
1385   GNUNET_assert (dc->th == NULL);
1386   dc->client = client;
1387   GNUNET_CLIENT_receive (client,
1388                          &receive_results,
1389                          dc,
1390                          GNUNET_TIME_UNIT_FOREVER_REL);
1391   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1392   GNUNET_FS_download_make_status_ (&pi, dc);
1393   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1394                                          &retry_entry,
1395                                          dc);
1396   dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1397                                                 sizeof (struct SearchMessage),
1398                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1399                                                 GNUNET_NO,
1400                                                 &transmit_download_request,
1401                                                 dc);
1402 }
1403
1404
1405 /**
1406  * We must stop to ask the FS service for our blocks.  Pause the download.
1407  *
1408  * @param cls the 'struct GNUNET_FS_DownloadContext'
1409  */
1410 static void
1411 deactivate_fs_download (void *cls)
1412 {
1413   struct GNUNET_FS_DownloadContext *dc = cls;
1414   struct GNUNET_FS_ProgressInfo pi;
1415
1416   if (NULL != dc->th)
1417     {
1418       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1419       dc->th = NULL;
1420     }
1421   if (NULL != dc->client)
1422     {
1423       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1424       dc->client = NULL;
1425     }
1426   pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1427   GNUNET_FS_download_make_status_ (&pi, dc);
1428 }
1429
1430
1431 /**
1432  * Create SUSPEND event for the given download operation
1433  * and then clean up our state (without stop signal).
1434  *
1435  * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
1436  */
1437 void
1438 GNUNET_FS_download_signal_suspend_ (void *cls)
1439 {
1440   struct GNUNET_FS_DownloadContext *dc = cls;
1441   struct GNUNET_FS_ProgressInfo pi;
1442
1443   if (dc->top != NULL)
1444     GNUNET_FS_end_top (dc->h, dc->top);
1445   while (NULL != dc->child_head)
1446     GNUNET_FS_download_signal_suspend_ (dc->child_head);
1447   if (dc->search != NULL)
1448     {
1449       dc->search->download = NULL;
1450       dc->search = NULL;
1451     }
1452   if (dc->job_queue != NULL)
1453     {
1454       GNUNET_FS_dequeue_ (dc->job_queue);
1455       dc->job_queue = NULL;
1456     }
1457   if (dc->parent != NULL)
1458     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1459                                  dc->parent->child_tail,
1460                                  dc);
1461   pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
1462   GNUNET_FS_download_make_status_ (&pi, dc);
1463   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1464     GNUNET_SCHEDULER_cancel (dc->h->sched,
1465                              dc->task);
1466   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1467                                          &free_entry,
1468                                          NULL);
1469   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1470   GNUNET_free_non_null (dc->filename);
1471   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1472   GNUNET_FS_uri_destroy (dc->uri);
1473   GNUNET_free_non_null (dc->temp_filename);
1474   GNUNET_free_non_null (dc->serialization);
1475   GNUNET_free (dc);
1476 }
1477
1478
1479 /**
1480  * Download parts of a file.  Note that this will store
1481  * the blocks at the respective offset in the given file.  Also, the
1482  * download is still using the blocking of the underlying FS
1483  * encoding.  As a result, the download may *write* outside of the
1484  * given boundaries (if offset and length do not match the 32k FS
1485  * block boundaries). <p>
1486  *
1487  * This function should be used to focus a download towards a
1488  * particular portion of the file (optimization), not to strictly
1489  * limit the download to exactly those bytes.
1490  *
1491  * @param h handle to the file sharing subsystem
1492  * @param uri the URI of the file (determines what to download); CHK or LOC URI
1493  * @param meta known metadata for the file (can be NULL)
1494  * @param filename where to store the file, maybe NULL (then no file is
1495  *        created on disk and data must be grabbed from the callbacks)
1496  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1497  *        can be NULL (in which case we will pick a name if needed); the temporary file
1498  *        may already exist, in which case we will try to use the data that is there and
1499  *        if it is not what is desired, will overwrite it
1500  * @param offset at what offset should we start the download (typically 0)
1501  * @param length how many bytes should be downloaded starting at offset
1502  * @param anonymity anonymity level to use for the download
1503  * @param options various options
1504  * @param cctx initial value for the client context for this download
1505  * @param parent parent download to associate this download with (use NULL
1506  *        for top-level downloads; useful for manually-triggered recursive downloads)
1507  * @return context that can be used to control this download
1508  */
1509 struct GNUNET_FS_DownloadContext *
1510 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
1511                           const struct GNUNET_FS_Uri *uri,
1512                           const struct GNUNET_CONTAINER_MetaData *meta,
1513                           const char *filename,
1514                           const char *tempname,
1515                           uint64_t offset,
1516                           uint64_t length,
1517                           uint32_t anonymity,
1518                           enum GNUNET_FS_DownloadOptions options,
1519                           void *cctx,
1520                           struct GNUNET_FS_DownloadContext *parent)
1521 {
1522   struct GNUNET_FS_ProgressInfo pi;
1523   struct GNUNET_FS_DownloadContext *dc;
1524
1525   GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
1526   if ( (offset + length < offset) ||
1527        (offset + length > uri->data.chk.file_length) )
1528     {
1529       GNUNET_break (0);
1530       return NULL;
1531     }
1532 #if DEBUG_DOWNLOAD
1533   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1534               "Starting download `%s' of %llu bytes\n",
1535               filename,
1536               (unsigned long long) length);
1537 #endif
1538   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1539   dc->h = h;
1540   dc->parent = parent;
1541   if (parent != NULL)
1542     {
1543       GNUNET_CONTAINER_DLL_insert (parent->child_head,
1544                                    parent->child_tail,
1545                                    dc);
1546     }
1547   dc->uri = GNUNET_FS_uri_dup (uri);
1548   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1549   dc->client_info = cctx;
1550   dc->start_time = GNUNET_TIME_absolute_get ();
1551   if (NULL != filename)
1552     {
1553       dc->filename = GNUNET_strdup (filename);
1554       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1555         GNUNET_DISK_file_size (filename,
1556                                &dc->old_file_size,
1557                                GNUNET_YES);
1558     }
1559   if (GNUNET_FS_uri_test_loc (dc->uri))
1560     GNUNET_assert (GNUNET_OK ==
1561                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1562                                                         &dc->target));
1563   dc->offset = offset;
1564   dc->length = length;
1565   dc->anonymity = anonymity;
1566   dc->options = options;
1567   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1568   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1569   if ( (filename == NULL) &&
1570        (is_recursive_download (dc) ) )
1571     {
1572       if (tempname != NULL)
1573         dc->temp_filename = GNUNET_strdup (tempname);
1574       else
1575         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1576     }
1577
1578 #if DEBUG_DOWNLOAD
1579   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1580               "Download tree has depth %u\n",
1581               dc->treedepth);
1582 #endif
1583   if (parent == NULL)
1584     {
1585       dc->top = GNUNET_FS_make_top (dc->h,
1586                                     &GNUNET_FS_download_signal_suspend_,
1587                                     dc);
1588     }
1589   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1590   pi.value.download.specifics.start.meta = meta;
1591   GNUNET_FS_download_make_status_ (&pi, dc);
1592   schedule_block_download (dc,
1593                            &dc->uri->data.chk.chk,
1594                            0,
1595                            1 /* 0 == CHK, 1 == top */);
1596   GNUNET_FS_download_sync_ (dc);
1597   GNUNET_FS_download_start_downloading_ (dc);
1598   return dc;
1599 }
1600
1601
1602 /**
1603  * Download parts of a file based on a search result.  The download
1604  * will be associated with the search result (and the association
1605  * will be preserved when serializing/deserializing the state).
1606  * If the search is stopped, the download will not be aborted but
1607  * be 'promoted' to a stand-alone download.
1608  *
1609  * As with the other download function, this will store
1610  * the blocks at the respective offset in the given file.  Also, the
1611  * download is still using the blocking of the underlying FS
1612  * encoding.  As a result, the download may *write* outside of the
1613  * given boundaries (if offset and length do not match the 32k FS
1614  * block boundaries). <p>
1615  *
1616  * The given range can be used to focus a download towards a
1617  * particular portion of the file (optimization), not to strictly
1618  * limit the download to exactly those bytes.
1619  *
1620  * @param h handle to the file sharing subsystem
1621  * @param sr the search result to use for the download (determines uri and
1622  *        meta data and associations)
1623  * @param filename where to store the file, maybe NULL (then no file is
1624  *        created on disk and data must be grabbed from the callbacks)
1625  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1626  *        can be NULL (in which case we will pick a name if needed); the temporary file
1627  *        may already exist, in which case we will try to use the data that is there and
1628  *        if it is not what is desired, will overwrite it
1629  * @param offset at what offset should we start the download (typically 0)
1630  * @param length how many bytes should be downloaded starting at offset
1631  * @param anonymity anonymity level to use for the download
1632  * @param options various download options
1633  * @param cctx initial value for the client context for this download
1634  * @return context that can be used to control this download
1635  */
1636 struct GNUNET_FS_DownloadContext *
1637 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
1638                                       struct GNUNET_FS_SearchResult *sr,
1639                                       const char *filename,
1640                                       const char *tempname,
1641                                       uint64_t offset,
1642                                       uint64_t length,
1643                                       uint32_t anonymity,
1644                                       enum GNUNET_FS_DownloadOptions options,
1645                                       void *cctx)
1646 {
1647   struct GNUNET_FS_ProgressInfo pi;
1648   struct GNUNET_FS_DownloadContext *dc;
1649
1650   if ( (sr != NULL) &&
1651        (sr->download != NULL) )
1652     {
1653       GNUNET_break (0);
1654       return NULL;
1655     }
1656   GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri));
1657   if ( (offset + length < offset) ||
1658        (offset + length > sr->uri->data.chk.file_length) )
1659     {
1660       GNUNET_break (0);
1661       return NULL;
1662     }
1663 #if DEBUG_DOWNLOAD
1664   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1665               "Starting download `%s' of %llu bytes\n",
1666               filename,
1667               (unsigned long long) length);
1668 #endif
1669   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1670   dc->h = h;
1671   dc->search = sr;
1672   sr->download = dc;
1673   if (sr->probe_ctx != NULL)
1674     {
1675       GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
1676       sr->probe_ctx = NULL;
1677     }
1678   dc->uri = GNUNET_FS_uri_dup (sr->uri);
1679   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
1680   dc->client_info = cctx;
1681   dc->start_time = GNUNET_TIME_absolute_get ();
1682   if (NULL != filename)
1683     {
1684       dc->filename = GNUNET_strdup (filename);
1685       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1686         GNUNET_DISK_file_size (filename,
1687                                &dc->old_file_size,
1688                                GNUNET_YES);
1689     }
1690   if (GNUNET_FS_uri_test_loc (dc->uri))
1691     GNUNET_assert (GNUNET_OK ==
1692                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1693                                                         &dc->target));
1694   dc->offset = offset;
1695   dc->length = length;
1696   dc->anonymity = anonymity;
1697   dc->options = options;
1698   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1699   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1700   if ( (filename == NULL) &&
1701        (is_recursive_download (dc) ) )
1702     {
1703       if (tempname != NULL)
1704         dc->temp_filename = GNUNET_strdup (tempname);
1705       else
1706         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1707     }
1708
1709 #if DEBUG_DOWNLOAD
1710   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1711               "Download tree has depth %u\n",
1712               dc->treedepth);
1713 #endif
1714   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1715   pi.value.download.specifics.start.meta = dc->meta;
1716   GNUNET_FS_download_make_status_ (&pi, dc);
1717   schedule_block_download (dc,
1718                            &dc->uri->data.chk.chk,
1719                            0,
1720                            1 /* 0 == CHK, 1 == top */);
1721   GNUNET_FS_download_sync_ (dc);
1722   GNUNET_FS_download_start_downloading_ (dc);
1723   return dc;
1724 }
1725
1726
1727 /**
1728  * Start the downloading process (by entering the queue).
1729  *
1730  * @param dc our download context
1731  */
1732 void
1733 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
1734 {
1735   GNUNET_assert (dc->job_queue == NULL);
1736   dc->job_queue = GNUNET_FS_queue_ (dc->h,
1737                                     &activate_fs_download,
1738                                     &deactivate_fs_download,
1739                                     dc,
1740                                     (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
1741 }
1742
1743
1744 /**
1745  * Stop a download (aborts if download is incomplete).
1746  *
1747  * @param dc handle for the download
1748  * @param do_delete delete files of incomplete downloads
1749  */
1750 void
1751 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
1752                          int do_delete)
1753 {
1754   struct GNUNET_FS_ProgressInfo pi;
1755   int have_children;
1756
1757   if (dc->top != NULL)
1758     GNUNET_FS_end_top (dc->h, dc->top);
1759   if (dc->search != NULL)
1760     {
1761       dc->search->download = NULL;
1762       dc->search = NULL;
1763     }
1764   if (dc->job_queue != NULL)
1765     {
1766       GNUNET_FS_dequeue_ (dc->job_queue);
1767       dc->job_queue = NULL;
1768     }
1769   have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
1770   while (NULL != dc->child_head)
1771     GNUNET_FS_download_stop (dc->child_head,
1772                              do_delete);
1773   if (dc->parent != NULL)
1774     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1775                                  dc->parent->child_tail,
1776                                  dc);
1777   if (dc->serialization != NULL)
1778     GNUNET_FS_remove_sync_file_ (dc->h,
1779                                  ( (dc->parent != NULL)  || (dc->search != NULL) )
1780                                  ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1781                                  : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
1782                                  dc->serialization);
1783   if ( (GNUNET_YES == have_children) &&
1784        (dc->parent == NULL) )
1785     GNUNET_FS_remove_sync_dir_ (dc->h,
1786                                 (dc->search != NULL)
1787                                 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1788                                 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
1789                                 dc->serialization);
1790   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
1791   GNUNET_FS_download_make_status_ (&pi, dc);
1792   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1793     GNUNET_SCHEDULER_cancel (dc->h->sched,
1794                              dc->task);
1795   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1796                                          &free_entry,
1797                                          NULL);
1798   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1799   if (dc->filename != NULL)
1800     {
1801       if ( (dc->completed != dc->length) &&
1802            (GNUNET_YES == do_delete) )
1803         {
1804           if (0 != UNLINK (dc->filename))
1805             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1806                                       "unlink",
1807                                       dc->filename);
1808         }
1809       GNUNET_free (dc->filename);
1810     }
1811   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1812   GNUNET_FS_uri_destroy (dc->uri);
1813   if (NULL != dc->temp_filename)
1814     {
1815       if (0 != UNLINK (dc->temp_filename))
1816         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1817                                   "unlink",
1818                                   dc->temp_filename);
1819       GNUNET_free (dc->temp_filename);
1820     }
1821   GNUNET_free_non_null (dc->serialization);
1822   GNUNET_free (dc);
1823 }
1824
1825 /* end of fs_download.c */