src/fs/fs_download.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file fs/fs_download.c
  22  * @brief download methods
  23  * @author Christian Grothoff
  24  *
  25  * TODO:
  26  * - location URI suppport (can wait, easy)
  27  * - different priority for scheduling probe downloads?
  28  * - check if iblocks can be computed from existing blocks (can wait, hard)
  29  */
  30 #include "platform.h"
  31 #include "gnunet_constants.h"
  32 #include "gnunet_fs_service.h"
  33 #include "fs.h"
  34 #include "fs_tree.h"
  35
  36 #define DEBUG_DOWNLOAD GNUNET_NO
  37
  38 /**
  39  * Determine if the given download (options and meta data) should cause
  40  * use to try to do a recursive download.
  41  */
  42 static int
  43 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
  44 {
  45   return  (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
  46     ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
  47       ( (dc->meta == NULL) &&
  48         ( (NULL == dc->filename) ||
  49           ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
  50             (NULL !=
  51              strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
  52                      GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
  53 }
  54
  55
  56 /**
  57  * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
  58  * only have to truncate the file once we're done).
  59  *
  60  * Given the offset of a block (with respect to the DBLOCKS) and its
  61  * depth, return the offset where we would store this block in the
  62  * file.
  63  *
  64  * @param fsize overall file size
  65  * @param off offset of the block in the file
  66  * @param depth depth of the block in the tree
  67  * @param treedepth maximum depth of the tree
  68  * @return off for DBLOCKS (depth == treedepth),
  69  *         otherwise an offset past the end
  70  *         of the file that does not overlap
  71  *         with the range for any other block
  72  */
  73 static uint64_t
  74 compute_disk_offset (uint64_t fsize,
  75                      uint64_t off,
  76                      unsigned int depth,
  77                      unsigned int treedepth)
  78 {
  79   unsigned int i;
  80   uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
  81   uint64_t loff; /* where do IBlocks for depth "i" start? */
  82   unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
  83
  84   if (depth == treedepth)
  85     return off;
  86   /* first IBlocks start at the end of file, rounded up
  87      to full DBLOCK_SIZE */
  88   loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
  89   lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
  90   GNUNET_assert (0 == (off % DBLOCK_SIZE));
  91   ioff = (off / DBLOCK_SIZE);
  92   for (i=treedepth-1;i>depth;i--)
  93     {
  94       loff += lsize;
  95       lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
  96       GNUNET_assert (lsize > 0);
  97       GNUNET_assert (0 == (ioff % CHK_PER_INODE));
  98       ioff /= CHK_PER_INODE;
  99     }
 100   return loff + ioff * sizeof (struct ContentHashKey);
 101 }
 102
 103
 104 /**
 105  * Given a file of the specified treedepth and a block at the given
 106  * offset and depth, calculate the offset for the CHK at the given
 107  * index.
 108  *
 109  * @param offset the offset of the first
 110  *        DBLOCK in the subtree of the
 111  *        identified IBLOCK
 112  * @param depth the depth of the IBLOCK in the tree
 113  * @param treedepth overall depth of the tree
 114  * @param k which CHK in the IBLOCK are we
 115  *        talking about
 116  * @return offset if k=0, otherwise an appropriately
 117  *         larger value (i.e., if depth = treedepth-1,
 118  *         the returned value should be offset+DBLOCK_SIZE)
 119  */
 120 static uint64_t
 121 compute_dblock_offset (uint64_t offset,
 122                        unsigned int depth,
 123                        unsigned int treedepth,
 124                        unsigned int k)
 125 {
 126   unsigned int i;
 127   uint64_t lsize; /* what is the size of the sum of all DBlocks
 128                      that a CHK at depth i corresponds to? */
 129
 130   if (depth == treedepth)
 131     return offset;
 132   lsize = DBLOCK_SIZE;
 133   for (i=treedepth-1;i>depth;i--)
 134     lsize *= CHK_PER_INODE;
 135   return offset + k * lsize;
 136 }
 137
 138
 139 /**
 140  * Fill in all of the generic fields for a download event and call the
 141  * callback.
 142  *
 143  * @param pi structure to fill in
 144  * @param dc overall download context
 145  */
 146 void
 147 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
 148                                  struct GNUNET_FS_DownloadContext *dc)
 149 {
 150   pi->value.download.dc = dc;
 151   pi->value.download.cctx
 152     = dc->client_info;
 153   pi->value.download.pctx
 154     = (dc->parent == NULL) ? NULL : dc->parent->client_info;
 155   pi->value.download.sctx
 156     = (dc->search == NULL) ? NULL : dc->search->client_info;
 157   pi->value.download.uri
 158     = dc->uri;
 159   pi->value.download.filename
 160     = dc->filename;
 161   pi->value.download.size
 162     = dc->length;
 163   pi->value.download.duration
 164     = GNUNET_TIME_absolute_get_duration (dc->start_time);
 165   pi->value.download.completed
 166     = dc->completed;
 167   pi->value.download.anonymity
 168     = dc->anonymity;
 169   pi->value.download.eta
 170     = GNUNET_TIME_calculate_eta (dc->start_time,
 171                                  dc->completed,
 172                                  dc->length);
 173   pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
 174   if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
 175     dc->client_info = dc->h->upcb (dc->h->upcb_cls,
 176                                    pi);
 177   else
 178     dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
 179                                                         pi);
 180 }
 181
 182 /**
 183  * We're ready to transmit a search request to the
 184  * file-sharing service.  Do it.  If there is
 185  * more than one request pending, try to send
 186  * multiple or request another transmission.
 187  *
 188  * @param cls closure
 189  * @param size number of bytes available in buf
 190  * @param buf where the callee should write the message
 191  * @return number of bytes written to buf
 192  */
 193 static size_t
 194 transmit_download_request (void *cls,
 195                            size_t size,
 196                            void *buf);
 197
 198
 199 /**
 200  * Closure for iterator processing results.
 201  */
 202 struct ProcessResultClosure
 203 {
 204
 205   /**
 206    * Hash of data.
 207    */
 208   GNUNET_HashCode query;
 209
 210   /**
 211    * Data found in P2P network.
 212    */
 213   const void *data;
 214
 215   /**
 216    * Our download context.
 217    */
 218   struct GNUNET_FS_DownloadContext *dc;
 219
 220   /**
 221    * Number of bytes in data.
 222    */
 223   size_t size;
 224
 225   /**
 226    * Type of data.
 227    */
 228   enum GNUNET_BLOCK_Type type;
 229
 230   /**
 231    * Flag to indicate if this block should be stored on disk.
 232    */
 233   int do_store;
 234
 235 };
 236
 237
 238 /**
 239  * Iterator over entries in the pending requests in the 'active' map for the
 240  * reply that we just got.
 241  *
 242  * @param cls closure (our 'struct ProcessResultClosure')
 243  * @param key query for the given value / request
 244  * @param value value in the hash map (a 'struct DownloadRequest')
 245  * @return GNUNET_YES (we should continue to iterate); unless serious error
 246  */
 247 static int
 248 process_result_with_request (void *cls,
 249                              const GNUNET_HashCode * key,
 250                              void *value);
 251
 252
 253
 254 /**
 255  * Schedule the download of the specified block in the tree.
 256  *
 257  * @param dc overall download this block belongs to
 258  * @param chk content-hash-key of the block
 259  * @param offset offset of the block in the file
 260  *         (for IBlocks, the offset is the lowest
 261  *          offset of any DBlock in the subtree under
 262  *          the IBlock)
 263  * @param depth depth of the block, 0 is the root of the tree
 264  */
 265 static void
 266 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
 267                          const struct ContentHashKey *chk,
 268                          uint64_t offset,
 269                          unsigned int depth)
 270 {
 271   struct DownloadRequest *sm;
 272   uint64_t total;
 273   uint64_t off;
 274   size_t len;
 275   char block[DBLOCK_SIZE];
 276   GNUNET_HashCode key;
 277   struct ProcessResultClosure prc;
 278   struct GNUNET_DISK_FileHandle *fh;
 279
 280 #if DEBUG_DOWNLOAD
 281   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 282               "Scheduling download at offset %llu and depth %u for `%s'\n",
 283               (unsigned long long) offset,
 284               depth,
 285               GNUNET_h2s (&chk->query));
 286 #endif
 287   total = GNUNET_ntohll (dc->uri->data.chk.file_length);
 288   off = compute_disk_offset (total,
 289                              offset,
 290                              depth,
 291                              dc->treedepth);
 292   len = GNUNET_FS_tree_calculate_block_size (total,
 293                                              dc->treedepth,
 294                                              offset,
 295                                              depth);
 296   sm = GNUNET_malloc (sizeof (struct DownloadRequest));
 297   sm->chk = *chk;
 298   sm->offset = offset;
 299   sm->depth = depth;
 300   sm->is_pending = GNUNET_YES;
 301   sm->next = dc->pending;
 302   dc->pending = sm;
 303   GNUNET_CONTAINER_multihashmap_put (dc->active,
 304                                      &chk->query,
 305                                      sm,
 306                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
 307   fh = NULL;
 308   if ( (dc->old_file_size > off) &&
 309        (dc->filename != NULL) )
 310     fh = GNUNET_DISK_file_open (dc->filename,
 311                                 GNUNET_DISK_OPEN_READ,
 312                                 GNUNET_DISK_PERM_NONE);
 313   if ( (fh != NULL) &&
 314        (off  ==
 315         GNUNET_DISK_file_seek (fh,
 316                                off,
 317                                GNUNET_DISK_SEEK_SET) ) &&
 318        (len ==
 319         GNUNET_DISK_file_read (fh,
 320                                block,
 321                                len)) )
 322     {
 323       GNUNET_CRYPTO_hash (block, len, &key);
 324       if (0 == memcmp (&key,
 325                        &chk->key,
 326                        sizeof (GNUNET_HashCode)))
 327         {
 328           char enc[len];
 329           struct GNUNET_CRYPTO_AesSessionKey sk;
 330           struct GNUNET_CRYPTO_AesInitializationVector iv;
 331           GNUNET_HashCode query;
 332
 333           GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
 334           if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
 335                                                &sk,
 336                                                &iv,
 337                                                enc))
 338             {
 339               GNUNET_break (0);
 340               goto do_download;
 341             }
 342           GNUNET_CRYPTO_hash (enc, len, &query);
 343           if (0 == memcmp (&query,
 344                            &chk->query,
 345                            sizeof (GNUNET_HashCode)))
 346             {
 347 #if DEBUG_DOWNLOAD
 348               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 349                           "Matching block already present, no need for download!\n");
 350 #endif
 351               /* already got it! */
 352               prc.dc = dc;
 353               prc.data = enc;
 354               prc.size = len;
 355               prc.type = (dc->treedepth == depth)
 356                 ? GNUNET_BLOCK_TYPE_DBLOCK
 357                 : GNUNET_BLOCK_TYPE_IBLOCK;
 358               prc.query = chk->query;
 359               prc.do_store = GNUNET_NO; /* useless */
 360               process_result_with_request (&prc,
 361                                            &key,
 362                                            sm);
 363             }
 364           else
 365             {
 366               GNUNET_break_op (0);
 367             }
 368           GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 369           return;
 370         }
 371     }
 372  do_download:
 373   if (fh != NULL)
 374     GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 375   if (depth < dc->treedepth)
 376     {
 377       // FIXME: try if we could
 378       // reconstitute this IBLOCK
 379       // from the existing blocks on disk (can wait)
 380       // (read block(s), encode, compare with
 381       // query; if matches, simply return)
 382     }
 383
 384   if ( (dc->th == NULL) &&
 385        (dc->client != NULL) )
 386     {
 387 #if DEBUG_DOWNLOAD
 388       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 389                   "Asking for transmission to FS service\n");
 390 #endif
 391       dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
 392                                                     sizeof (struct SearchMessage),
 393                                                     GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 394                                                     GNUNET_NO,
 395                                                     &transmit_download_request,
 396                                                     dc);
 397     }
 398   else
 399     {
 400 #if DEBUG_DOWNLOAD
 401       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 402                   "Transmission request not issued (%p %p)\n",
 403                   dc->th,
 404                   dc->client);
 405 #endif
 406
 407     }
 408
 409 }
 410
 411
 412
 413 /**
 414  * Suggest a filename based on given metadata.
 415  *
 416  * @param md given meta data
 417  * @return NULL if meta data is useless for suggesting a filename
 418  */
 419 char *
 420 GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md)
 421 {
 422   static const char *mimeMap[][2] = {
 423     {"application/bz2", ".bz2"},
 424     {"application/gnunet-directory", ".gnd"},
 425     {"application/java", ".class"},
 426     {"application/msword", ".doc"},
 427     {"application/ogg", ".ogg"},
 428     {"application/pdf", ".pdf"},
 429     {"application/pgp-keys", ".key"},
 430     {"application/pgp-signature", ".pgp"},
 431     {"application/postscript", ".ps"},
 432     {"application/rar", ".rar"},
 433     {"application/rtf", ".rtf"},
 434     {"application/xml", ".xml"},
 435     {"application/x-debian-package", ".deb"},
 436     {"application/x-dvi", ".dvi"},
 437     {"applixation/x-flac", ".flac"},
 438     {"applixation/x-gzip", ".gz"},
 439     {"application/x-java-archive", ".jar"},
 440     {"application/x-java-vm", ".class"},
 441     {"application/x-python-code", ".pyc"},
 442     {"application/x-redhat-package-manager", ".rpm"},
 443     {"application/x-rpm", ".rpm"},
 444     {"application/x-tar", ".tar"},
 445     {"application/x-tex-pk", ".pk"},
 446     {"application/x-texinfo", ".texinfo"},
 447     {"application/x-xcf", ".xcf"},
 448     {"application/x-xfig", ".xfig"},
 449     {"application/zip", ".zip"},
 450
 451     {"audio/midi", ".midi"},
 452     {"audio/mpeg", ".mp3"},
 453     {"audio/real", ".rm"},
 454     {"audio/x-wav", ".wav"},
 455
 456     {"image/gif", ".gif"},
 457     {"image/jpeg", ".jpg"},
 458     {"image/pcx", ".pcx"},
 459     {"image/png", ".png"},
 460     {"image/tiff", ".tiff"},
 461     {"image/x-ms-bmp", ".bmp"},
 462     {"image/x-xpixmap", ".xpm"},
 463
 464     {"text/css", ".css"},
 465     {"text/html", ".html"},
 466     {"text/plain", ".txt"},
 467     {"text/rtf", ".rtf"},
 468     {"text/x-c++hdr", ".h++"},
 469     {"text/x-c++src", ".c++"},
 470     {"text/x-chdr", ".h"},
 471     {"text/x-csrc", ".c"},
 472     {"text/x-java", ".java"},
 473     {"text/x-moc", ".moc"},
 474     {"text/x-pascal", ".pas"},
 475     {"text/x-perl", ".pl"},
 476     {"text/x-python", ".py"},
 477     {"text/x-tex", ".tex"},
 478
 479     {"video/avi", ".avi"},
 480     {"video/mpeg", ".mpeg"},
 481     {"video/quicktime", ".qt"},
 482     {"video/real", ".rm"},
 483     {"video/x-msvideo", ".avi"},
 484     {NULL, NULL},
 485   };
 486   char *ret;
 487   unsigned int i;
 488   char *mime;
 489   char *base;
 490   const char *ext;
 491
 492   ret = GNUNET_CONTAINER_meta_data_get_by_type (md,
 493                                                 EXTRACTOR_METATYPE_FILENAME);
 494   if (ret != NULL)
 495     return ret;
 496   ext = NULL;
 497   mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
 498                                                  EXTRACTOR_METATYPE_MIMETYPE);
 499   if (mime != NULL)
 500     {
 501       i = 0;
 502       while ( (mimeMap[i][0] != NULL) &&
 503               (0 != strcmp (mime, mimeMap[i][0])))
 504         i++;
 505       if (mimeMap[i][1] == NULL)
 506         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG |
 507                     GNUNET_ERROR_TYPE_BULK,
 508                     _("Did not find mime type `%s' in extension list.\n"),
 509                     mime);
 510       else
 511         ext = mimeMap[i][1];
 512       GNUNET_free (mime);
 513     }
 514   base = GNUNET_CONTAINER_meta_data_get_first_by_types (md,
 515                                                         EXTRACTOR_METATYPE_TITLE,
 516                                                         EXTRACTOR_METATYPE_BOOK_TITLE,
 517                                                         EXTRACTOR_METATYPE_ORIGINAL_TITLE,
 518                                                         EXTRACTOR_METATYPE_PACKAGE_NAME,
 519                                                         EXTRACTOR_METATYPE_URL,
 520                                                         EXTRACTOR_METATYPE_URI,
 521                                                         EXTRACTOR_METATYPE_DESCRIPTION,
 522                                                         EXTRACTOR_METATYPE_ISRC,
 523                                                         EXTRACTOR_METATYPE_JOURNAL_NAME,
 524                                                         EXTRACTOR_METATYPE_AUTHOR_NAME,
 525                                                         EXTRACTOR_METATYPE_SUBJECT,
 526                                                         EXTRACTOR_METATYPE_ALBUM,
 527                                                         EXTRACTOR_METATYPE_ARTIST,
 528                                                         EXTRACTOR_METATYPE_KEYWORDS,
 529                                                         EXTRACTOR_METATYPE_COMMENT,
 530                                                         EXTRACTOR_METATYPE_UNKNOWN,
 531                                                         -1);
 532   if ( (base == NULL) &&
 533        (ext == NULL) )
 534     return NULL;
 535   if (base == NULL)
 536     return GNUNET_strdup (ext);
 537   if (ext == NULL)
 538     return base;
 539   GNUNET_asprintf (&ret,
 540                    "%s%s",
 541                    base,
 542                    ext);
 543   GNUNET_free (base);
 544   return ret;
 545 }
 546
 547
 548 /**
 549  * We've lost our connection with the FS service.
 550  * Re-establish it and re-transmit all of our
 551  * pending requests.
 552  *
 553  * @param dc download context that is having trouble
 554  */
 555 static void
 556 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 557
 558
 559 /**
 560  * We found an entry in a directory.  Check if the respective child
 561  * already exists and if not create the respective child download.
 562  *
 563  * @param cls the parent download
 564  * @param filename name of the file in the directory
 565  * @param uri URI of the file (CHK or LOC)
 566  * @param meta meta data of the file
 567  * @param length number of bytes in data
 568  * @param data contents of the file (or NULL if they were not inlined)
 569  */
 570 static void
 571 trigger_recursive_download (void *cls,
 572                             const char *filename,
 573                             const struct GNUNET_FS_Uri *uri,
 574                             const struct GNUNET_CONTAINER_MetaData *meta,
 575                             size_t length,
 576                             const void *data);
 577
 578
 579 /**
 580  * We're done downloading a directory.  Open the file and
 581  * trigger all of the (remaining) child downloads.
 582  *
 583  * @param dc context of download that just completed
 584  */
 585 static void
 586 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 587 {
 588   size_t size;
 589   uint64_t size64;
 590   void *data;
 591   struct GNUNET_DISK_FileHandle *h;
 592   struct GNUNET_DISK_MapHandle *m;
 593
 594   size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
 595   size = (size_t) size64;
 596   if (size64 != (uint64_t) size)
 597     {
 598       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 599                   _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
 600       return;
 601     }
 602   if (dc->filename != NULL)
 603     {
 604       h = GNUNET_DISK_file_open (dc->filename,
 605                                  GNUNET_DISK_OPEN_READ,
 606                                  GNUNET_DISK_PERM_NONE);
 607     }
 608   else
 609     {
 610       GNUNET_assert (dc->temp_filename != NULL);
 611       h = GNUNET_DISK_file_open (dc->temp_filename,
 612                                  GNUNET_DISK_OPEN_READ,
 613                                  GNUNET_DISK_PERM_NONE);
 614     }
 615   if (h == NULL)
 616     return; /* oops */
 617   data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
 618   if (data == NULL)
 619     {
 620       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 621                   _("Directory too large for system address space\n"));
 622     }
 623   else
 624     {
 625       GNUNET_FS_directory_list_contents (size,
 626                                          data,
 627                                          0,
 628                                          &trigger_recursive_download,
 629                                          dc);
 630       GNUNET_DISK_file_unmap (m);
 631     }
 632   GNUNET_DISK_file_close (h);
 633   if (dc->filename == NULL)
 634     {
 635       if (0 != UNLINK (dc->temp_filename))
 636         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
 637                                   "unlink",
 638                                   dc->temp_filename);
 639       GNUNET_free (dc->temp_filename);
 640       dc->temp_filename = NULL;
 641     }
 642 }
 643
 644
 645 /**
 646  * Check if all child-downloads have completed and
 647  * if so, signal completion (and possibly recurse to
 648  * parent).
 649  */
 650 static void
 651 check_completed (struct GNUNET_FS_DownloadContext *dc)
 652 {
 653   struct GNUNET_FS_ProgressInfo pi;
 654   struct GNUNET_FS_DownloadContext *pos;
 655
 656   pos = dc->child_head;
 657   while (pos != NULL)
 658     {
 659       if ( (pos->emsg == NULL) &&
 660            (pos->completed < pos->length) )
 661         return; /* not done yet */
 662       if ( (pos->child_head != NULL) &&
 663            (pos->has_finished != GNUNET_YES) )
 664         return; /* not transitively done yet */
 665       pos = pos->next;
 666     }
 667   dc->has_finished = GNUNET_YES;
 668   GNUNET_FS_download_sync_ (dc);
 669   /* signal completion */
 670   pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
 671   GNUNET_FS_download_make_status_ (&pi, dc);
 672   if (dc->parent != NULL)
 673     check_completed (dc->parent);
 674 }
 675
 676
 677 /**
 678  * We found an entry in a directory.  Check if the respective child
 679  * already exists and if not create the respective child download.
 680  *
 681  * @param cls the parent download
 682  * @param filename name of the file in the directory
 683  * @param uri URI of the file (CHK or LOC)
 684  * @param meta meta data of the file
 685  * @param length number of bytes in data
 686  * @param data contents of the file (or NULL if they were not inlined)
 687  */
 688 static void
 689 trigger_recursive_download (void *cls,
 690                             const char *filename,
 691                             const struct GNUNET_FS_Uri *uri,
 692                             const struct GNUNET_CONTAINER_MetaData *meta,
 693                             size_t length,
 694                             const void *data)
 695 {
 696   struct GNUNET_FS_DownloadContext *dc = cls;
 697   struct GNUNET_FS_DownloadContext *cpos;
 698   struct GNUNET_DISK_FileHandle *fh;
 699   char *temp_name;
 700   const char *real_name;
 701   char *fn;
 702   char *us;
 703   char *ext;
 704   char *dn;
 705   char *pos;
 706   char *full_name;
 707
 708   if (NULL == uri)
 709     return; /* entry for the directory itself */
 710   cpos = dc->child_head;
 711   while (cpos != NULL)
 712     {
 713       if ( (GNUNET_FS_uri_test_equal (uri,
 714                                       cpos->uri)) ||
 715            ( (filename != NULL) &&
 716              (0 == strcmp (cpos->filename,
 717                            filename)) ) )
 718         break;
 719       cpos = cpos->next;
 720     }
 721   if (cpos != NULL)
 722     return; /* already exists */
 723   fn = NULL;
 724   if (NULL == filename)
 725     {
 726       fn = GNUNET_FS_meta_data_suggest_filename (meta);
 727       if (fn == NULL)
 728         {
 729           us = GNUNET_FS_uri_to_string (uri);
 730           fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_PREFIX
 731                                            GNUNET_FS_URI_CHK_INFIX)]);
 732           GNUNET_free (us);
 733         }
 734       else if (fn[0] == '.')
 735         {
 736           ext = fn;
 737           us = GNUNET_FS_uri_to_string (uri);
 738           GNUNET_asprintf (&fn,
 739                            "%s%s",
 740                            &us[strlen (GNUNET_FS_URI_PREFIX
 741                                        GNUNET_FS_URI_CHK_INFIX)], ext);
 742           GNUNET_free (ext);
 743           GNUNET_free (us);
 744         }
 745       /* change '\' to '/' (this should have happened
 746        during insertion, but malicious peers may
 747        not have done this) */
 748       while (NULL != (pos = strstr (fn, "\\")))
 749         *pos = '/';
 750       /* remove '../' everywhere (again, well-behaved
 751          peers don't do this, but don't trust that
 752          we did not get something nasty) */
 753       while (NULL != (pos = strstr (fn, "../")))
 754         {
 755           pos[0] = '_';
 756           pos[1] = '_';
 757           pos[2] = '_';
 758         }
 759       filename = fn;
 760     }
 761   if (dc->filename == NULL)
 762     {
 763       full_name = NULL;
 764     }
 765   else
 766     {
 767       dn = GNUNET_strdup (dc->filename);
 768       GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 769                      (NULL !=
 770                       strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 771                               GNUNET_FS_DIRECTORY_EXT)) );
 772       if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 773            (NULL !=
 774             strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 775                     GNUNET_FS_DIRECTORY_EXT)) )
 776         dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
 777       if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
 778            ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
 779              (NULL ==
 780               strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
 781                       GNUNET_FS_DIRECTORY_EXT)) ) )
 782         {
 783           GNUNET_asprintf (&full_name,
 784                            "%s%s%s%s",
 785                            dn,
 786                            DIR_SEPARATOR_STR,
 787                            filename,
 788                            GNUNET_FS_DIRECTORY_EXT);
 789         }
 790       else
 791         {
 792           GNUNET_asprintf (&full_name,
 793                            "%s%s%s",
 794                            dn,
 795                            DIR_SEPARATOR_STR,
 796                            filename);
 797         }
 798       GNUNET_free (dn);
 799     }
 800   if ( (full_name != NULL) &&
 801        (GNUNET_OK !=
 802         GNUNET_DISK_directory_create_for_file (full_name)) )
 803     {
 804       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 805                   _("Failed to create directory for recursive download of `%s'\n"),
 806                   full_name);
 807       GNUNET_free (full_name);
 808       GNUNET_free_non_null (fn);
 809       return;
 810     }
 811
 812   temp_name = NULL;
 813   if ( (data != NULL) &&
 814        (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
 815     {
 816       if (full_name == NULL)
 817         {
 818           temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
 819           real_name = temp_name;
 820         }
 821       else
 822         {
 823           real_name = full_name;
 824         }
 825       /* write to disk, then trigger normal download which will instantly progress to completion */
 826       fh = GNUNET_DISK_file_open (real_name,
 827                                   GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
 828                                   GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
 829       if (fh == NULL)
 830         {
 831           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 832                                     "open",
 833                                     real_name);
 834           GNUNET_free (full_name);
 835           GNUNET_free_non_null (fn);
 836           return;
 837         }
 838       if (length !=
 839           GNUNET_DISK_file_write (fh,
 840                                   data,
 841                                   length))
 842         {
 843           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 844                                     "write",
 845                                     full_name);
 846         }
 847       GNUNET_DISK_file_close (fh);
 848     }
 849   GNUNET_FS_download_start (dc->h,
 850                             uri,
 851                             meta,
 852                             full_name, temp_name,
 853                             0,
 854                             GNUNET_FS_uri_chk_get_file_size (uri),
 855                             dc->anonymity,
 856                             dc->options,
 857                             NULL,
 858                             dc);
 859   GNUNET_free_non_null (full_name);
 860   GNUNET_free_non_null (temp_name);
 861   GNUNET_free_non_null (fn);
 862 }
 863
 864
 865 /**
 866  * Free entries in the map.
 867  *
 868  * @param cls unused (NULL)
 869  * @param key unused
 870  * @param entry entry of type "struct DownloadRequest" which is freed
 871  * @return GNUNET_OK
 872  */
 873 static int
 874 free_entry (void *cls,
 875             const GNUNET_HashCode *key,
 876             void *entry)
 877 {
 878   GNUNET_free (entry);
 879   return GNUNET_OK;
 880 }
 881
 882
 883 /**
 884  * Iterator over entries in the pending requests in the 'active' map for the
 885  * reply that we just got.
 886  *
 887  * @param cls closure (our 'struct ProcessResultClosure')
 888  * @param key query for the given value / request
 889  * @param value value in the hash map (a 'struct DownloadRequest')
 890  * @return GNUNET_YES (we should continue to iterate); unless serious error
 891  */
 892 static int
 893 process_result_with_request (void *cls,
 894                              const GNUNET_HashCode * key,
 895                              void *value)
 896 {
 897   struct ProcessResultClosure *prc = cls;
 898   struct DownloadRequest *sm = value;
 899   struct DownloadRequest *ppos;
 900   struct DownloadRequest *pprev;
 901   struct GNUNET_DISK_FileHandle *fh;
 902   struct GNUNET_FS_DownloadContext *dc = prc->dc;
 903   struct GNUNET_CRYPTO_AesSessionKey skey;
 904   struct GNUNET_CRYPTO_AesInitializationVector iv;
 905   char pt[prc->size];
 906   struct GNUNET_FS_ProgressInfo pi;
 907   uint64_t off;
 908   size_t bs;
 909   size_t app;
 910   int i;
 911   struct ContentHashKey *chk;
 912
 913   fh = NULL;
 914   bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
 915                                             dc->treedepth,
 916                                             sm->offset,
 917                                             sm->depth);
 918   if (prc->size != bs)
 919     {
 920 #if DEBUG_DOWNLOAD
 921       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 922                   "Internal error or bogus download URI (expected %u bytes, got %u)\n",
 923                   bs,
 924                   prc->size);
 925 #endif
 926       dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
 927       goto signal_error;
 928     }
 929   GNUNET_assert (GNUNET_YES ==
 930                  GNUNET_CONTAINER_multihashmap_remove (dc->active,
 931                                                        &prc->query,
 932                                                        sm));
 933   /* if this request is on the pending list, remove it! */
 934   pprev = NULL;
 935   ppos = dc->pending;
 936   while (ppos != NULL)
 937     {
 938       if (ppos == sm)
 939         {
 940           if (pprev == NULL)
 941             dc->pending = ppos->next;
 942           else
 943             pprev->next = ppos->next;
 944           break;
 945         }
 946       pprev = ppos;
 947       ppos = ppos->next;
 948     }
 949   GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
 950   if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
 951                                        prc->size,
 952                                        &skey,
 953                                        &iv,
 954                                        pt))
 955     {
 956       GNUNET_break (0);
 957       dc->emsg = GNUNET_strdup ("internal error decrypting content");
 958       goto signal_error;
 959     }
 960   off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
 961                              sm->offset,
 962                              sm->depth,
 963                              dc->treedepth);
 964   /* save to disk */
 965   if ( ( GNUNET_YES == prc->do_store) &&
 966        ( (dc->filename != NULL) ||
 967          (is_recursive_download (dc)) ) &&
 968        ( (sm->depth == dc->treedepth) ||
 969          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 970     {
 971       fh = GNUNET_DISK_file_open (dc->filename != NULL
 972                                   ? dc->filename
 973                                   : dc->temp_filename,
 974                                   GNUNET_DISK_OPEN_READWRITE |
 975                                   GNUNET_DISK_OPEN_CREATE,
 976                                   GNUNET_DISK_PERM_USER_READ |
 977                                   GNUNET_DISK_PERM_USER_WRITE |
 978                                   GNUNET_DISK_PERM_GROUP_READ |
 979                                   GNUNET_DISK_PERM_OTHER_READ);
 980     }
 981   if ( (NULL == fh) &&
 982        (GNUNET_YES == prc->do_store) &&
 983        ( (dc->filename != NULL) ||
 984          (is_recursive_download (dc)) ) &&
 985        ( (sm->depth == dc->treedepth) ||
 986          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 987     {
 988       GNUNET_asprintf (&dc->emsg,
 989                        _("Download failed: could not open file `%s': %s\n"),
 990                        dc->filename,
 991                        STRERROR (errno));
 992       goto signal_error;
 993     }
 994   if (fh != NULL)
 995     {
 996 #if DEBUG_DOWNLOAD
 997       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 998                   "Saving decrypted block to disk at offset %llu\n",
 999                   (unsigned long long) off);
1000 #endif
1001       if ( (off  !=
1002             GNUNET_DISK_file_seek (fh,
1003                                    off,
1004                                    GNUNET_DISK_SEEK_SET) ) )
1005         {
1006           GNUNET_asprintf (&dc->emsg,
1007                            _("Failed to seek to offset %llu in file `%s': %s\n"),
1008                            (unsigned long long) off,
1009                            dc->filename,
1010                            STRERROR (errno));
1011           goto signal_error;
1012         }
1013       if (prc->size !=
1014           GNUNET_DISK_file_write (fh,
1015                                   pt,
1016                                   prc->size))
1017         {
1018           GNUNET_asprintf (&dc->emsg,
1019                            _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1020                            (unsigned int) prc->size,
1021                            (unsigned long long) off,
1022                            dc->filename,
1023                            STRERROR (errno));
1024           goto signal_error;
1025         }
1026       GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1027       fh = NULL;
1028     }
1029   if (sm->depth == dc->treedepth)
1030     {
1031       app = prc->size;
1032       if (sm->offset < dc->offset)
1033         {
1034           /* starting offset begins in the middle of pt,
1035              do not count first bytes as progress */
1036           GNUNET_assert (app > (dc->offset - sm->offset));
1037           app -= (dc->offset - sm->offset);
1038         }
1039       if (sm->offset + prc->size > dc->offset + dc->length)
1040         {
1041           /* end of block is after relevant range,
1042              do not count last bytes as progress */
1043           GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length));
1044           app -= (sm->offset + prc->size) - (dc->offset + dc->length);
1045         }
1046       dc->completed += app;
1047
1048       /* do recursive download if option is set and either meta data
1049          says it is a directory or if no meta data is given AND filename
1050          ends in '.gnd' (top-level case) */
1051       if (is_recursive_download (dc))
1052         GNUNET_FS_directory_list_contents (prc->size,
1053                                            pt,
1054                                            off,
1055                                            &trigger_recursive_download,
1056                                            dc);
1057
1058     }
1059   pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1060   pi.value.download.specifics.progress.data = pt;
1061   pi.value.download.specifics.progress.offset = sm->offset;
1062   pi.value.download.specifics.progress.data_len = prc->size;
1063   pi.value.download.specifics.progress.depth = sm->depth;
1064   GNUNET_FS_download_make_status_ (&pi, dc);
1065   GNUNET_assert (dc->completed <= dc->length);
1066   if (dc->completed == dc->length)
1067     {
1068 #if DEBUG_DOWNLOAD
1069       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1070                   "Download completed, truncating file to desired length %llu\n",
1071                   (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1072 #endif
1073       /* truncate file to size (since we store IBlocks at the end) */
1074       if (dc->filename != NULL)
1075         {
1076           if (0 != truncate (dc->filename,
1077                              GNUNET_ntohll (dc->uri->data.chk.file_length)))
1078             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1079                                       "truncate",
1080                                       dc->filename);
1081         }
1082       if (dc->job_queue != NULL)
1083         {
1084           GNUNET_FS_dequeue_ (dc->job_queue);
1085           dc->job_queue = NULL;
1086         }
1087       if (is_recursive_download (dc))
1088         full_recursive_download (dc);
1089       if (dc->child_head == NULL)
1090         {
1091           /* signal completion */
1092           pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
1093           GNUNET_FS_download_make_status_ (&pi, dc);
1094           if (dc->parent != NULL)
1095             check_completed (dc->parent);
1096         }
1097       GNUNET_assert (sm->depth == dc->treedepth);
1098     }
1099   if (sm->depth == dc->treedepth)
1100     {
1101       GNUNET_FS_download_sync_ (dc);
1102       GNUNET_free (sm);
1103       return GNUNET_YES;
1104     }
1105 #if DEBUG_DOWNLOAD
1106   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1107               "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1108               sm->depth,
1109               (unsigned long long) sm->offset);
1110 #endif
1111   GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1112   chk = (struct ContentHashKey*) pt;
1113   for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1114     {
1115       off = compute_dblock_offset (sm->offset,
1116                                    sm->depth,
1117                                    dc->treedepth,
1118                                    i);
1119       if ( (off + DBLOCK_SIZE >= dc->offset) &&
1120            (off < dc->offset + dc->length) )
1121         schedule_block_download (dc,
1122                                  &chk[i],
1123                                  off,
1124                                  sm->depth + 1);
1125     }
1126   GNUNET_free (sm);
1127   GNUNET_FS_download_sync_ (dc);
1128   return GNUNET_YES;
1129
1130  signal_error:
1131   if (fh != NULL)
1132     GNUNET_DISK_file_close (fh);
1133   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1134   pi.value.download.specifics.error.message = dc->emsg;
1135   GNUNET_FS_download_make_status_ (&pi, dc);
1136   /* abort all pending requests */
1137   if (NULL != dc->th)
1138     {
1139       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1140       dc->th = NULL;
1141     }
1142   GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1143   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1144                                          &free_entry,
1145                                          NULL);
1146   dc->pending = NULL;
1147   dc->client = NULL;
1148   GNUNET_free (sm);
1149   GNUNET_FS_download_sync_ (dc);
1150   return GNUNET_NO;
1151 }
1152
1153
1154 /**
1155  * Process a download result.
1156  *
1157  * @param dc our download context
1158  * @param type type of the result
1159  * @param data the (encrypted) response
1160  * @param size size of data
1161  */
1162 static void
1163 process_result (struct GNUNET_FS_DownloadContext *dc,
1164                 enum GNUNET_BLOCK_Type type,
1165                 const void *data,
1166                 size_t size)
1167 {
1168   struct ProcessResultClosure prc;
1169
1170   prc.dc = dc;
1171   prc.data = data;
1172   prc.size = size;
1173   prc.type = type;
1174   prc.do_store = GNUNET_YES;
1175   GNUNET_CRYPTO_hash (data, size, &prc.query);
1176 #if DEBUG_DOWNLOAD
1177   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1178               "Received result for query `%s' from `%s'-service\n",
1179               GNUNET_h2s (&prc.query),
1180               "FS");
1181 #endif
1182   GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1183                                               &prc.query,
1184                                               &process_result_with_request,
1185                                               &prc);
1186 }
1187
1188
1189 /**
1190  * Type of a function to call when we receive a message
1191  * from the service.
1192  *
1193  * @param cls closure
1194  * @param msg message received, NULL on timeout or fatal error
1195  */
1196 static void
1197 receive_results (void *cls,
1198                  const struct GNUNET_MessageHeader * msg)
1199 {
1200   struct GNUNET_FS_DownloadContext *dc = cls;
1201   const struct PutMessage *cm;
1202   uint16_t msize;
1203
1204   if ( (NULL == msg) ||
1205        (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1206        (sizeof (struct PutMessage) > ntohs(msg->size)) )
1207     {
1208       GNUNET_break (msg == NULL);
1209       try_reconnect (dc);
1210       return;
1211     }
1212   msize = ntohs(msg->size);
1213   cm = (const struct PutMessage*) msg;
1214   process_result (dc,
1215                   ntohl (cm->type),
1216                   &cm[1],
1217                   msize - sizeof (struct PutMessage));
1218   if (dc->client == NULL)
1219     return; /* fatal error */
1220   /* continue receiving */
1221   GNUNET_CLIENT_receive (dc->client,
1222                          &receive_results,
1223                          dc,
1224                          GNUNET_TIME_UNIT_FOREVER_REL);
1225 }
1226
1227
1228
1229 /**
1230  * We're ready to transmit a search request to the
1231  * file-sharing service.  Do it.  If there is
1232  * more than one request pending, try to send
1233  * multiple or request another transmission.
1234  *
1235  * @param cls closure
1236  * @param size number of bytes available in buf
1237  * @param buf where the callee should write the message
1238  * @return number of bytes written to buf
1239  */
1240 static size_t
1241 transmit_download_request (void *cls,
1242                            size_t size,
1243                            void *buf)
1244 {
1245   struct GNUNET_FS_DownloadContext *dc = cls;
1246   size_t msize;
1247   struct SearchMessage *sm;
1248
1249   dc->th = NULL;
1250   if (NULL == buf)
1251     {
1252 #if DEBUG_DOWNLOAD
1253       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1254                   "Transmitting download request failed, trying to reconnect\n");
1255 #endif
1256       try_reconnect (dc);
1257       return 0;
1258     }
1259   GNUNET_assert (size >= sizeof (struct SearchMessage));
1260   msize = 0;
1261   sm = buf;
1262   while ( (dc->pending != NULL) &&
1263           (size >= msize + sizeof (struct SearchMessage)) )
1264     {
1265 #if DEBUG_DOWNLOAD
1266       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1267                   "Transmitting download request for `%s' to `%s'-service\n",
1268                   GNUNET_h2s (&dc->pending->chk.query),
1269                   "FS");
1270 #endif
1271       memset (sm, 0, sizeof (struct SearchMessage));
1272       sm->header.size = htons (sizeof (struct SearchMessage));
1273       sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1274       if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1275         sm->options = htonl (1);
1276       else
1277         sm->options = htonl (0);
1278       if (dc->pending->depth == dc->treedepth)
1279         sm->type = htonl (GNUNET_BLOCK_TYPE_DBLOCK);
1280       else
1281         sm->type = htonl (GNUNET_BLOCK_TYPE_IBLOCK);
1282       sm->anonymity_level = htonl (dc->anonymity);
1283       sm->target = dc->target.hashPubKey;
1284       sm->query = dc->pending->chk.query;
1285       dc->pending->is_pending = GNUNET_NO;
1286       dc->pending = dc->pending->next;
1287       msize += sizeof (struct SearchMessage);
1288       sm++;
1289     }
1290   if (dc->pending != NULL)
1291     dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1292                                                   sizeof (struct SearchMessage),
1293                                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1294                                                   GNUNET_NO,
1295                                                   &transmit_download_request,
1296                                                   dc);
1297   return msize;
1298 }
1299
1300
1301 /**
1302  * Reconnect to the FS service and transmit our queries NOW.
1303  *
1304  * @param cls our download context
1305  * @param tc unused
1306  */
1307 static void
1308 do_reconnect (void *cls,
1309               const struct GNUNET_SCHEDULER_TaskContext *tc)
1310 {
1311   struct GNUNET_FS_DownloadContext *dc = cls;
1312   struct GNUNET_CLIENT_Connection *client;
1313
1314   dc->task = GNUNET_SCHEDULER_NO_TASK;
1315   client = GNUNET_CLIENT_connect (dc->h->sched,
1316                                   "fs",
1317                                   dc->h->cfg);
1318   if (NULL == client)
1319     {
1320       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1321                   "Connecting to `%s'-service failed, will try again.\n",
1322                   "FS");
1323       try_reconnect (dc);
1324       return;
1325     }
1326   dc->client = client;
1327   dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1328                                                 sizeof (struct SearchMessage),
1329                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1330                                                 GNUNET_NO,
1331                                                 &transmit_download_request,
1332                                                 dc);
1333   GNUNET_CLIENT_receive (client,
1334                          &receive_results,
1335                          dc,
1336                          GNUNET_TIME_UNIT_FOREVER_REL);
1337 }
1338
1339
1340 /**
1341  * Add entries that are not yet pending back to the pending list.
1342  *
1343  * @param cls our download context
1344  * @param key unused
1345  * @param entry entry of type "struct DownloadRequest"
1346  * @return GNUNET_OK
1347  */
1348 static int
1349 retry_entry (void *cls,
1350              const GNUNET_HashCode *key,
1351              void *entry)
1352 {
1353   struct GNUNET_FS_DownloadContext *dc = cls;
1354   struct DownloadRequest *dr = entry;
1355
1356   if (! dr->is_pending)
1357     {
1358       dr->next = dc->pending;
1359       dr->is_pending = GNUNET_YES;
1360       dc->pending = entry;
1361     }
1362   return GNUNET_OK;
1363 }
1364
1365
1366 /**
1367  * We've lost our connection with the FS service.
1368  * Re-establish it and re-transmit all of our
1369  * pending requests.
1370  *
1371  * @param dc download context that is having trouble
1372  */
1373 static void
1374 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1375 {
1376
1377   if (NULL != dc->client)
1378     {
1379 #if DEBUG_DOWNLOAD
1380       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1381                   "Moving all requests back to pending list\n");
1382 #endif
1383       if (NULL != dc->th)
1384         {
1385           GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1386           dc->th = NULL;
1387         }
1388       GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1389                                              &retry_entry,
1390                                              dc);
1391       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1392       dc->client = NULL;
1393     }
1394 #if DEBUG_DOWNLOAD
1395   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1396               "Will try to reconnect in 1s\n");
1397 #endif
1398   dc->task
1399     = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
1400                                     GNUNET_TIME_UNIT_SECONDS,
1401                                     &do_reconnect,
1402                                     dc);
1403 }
1404
1405
1406
1407 /**
1408  * We're allowed to ask the FS service for our blocks.  Start the download.
1409  *
1410  * @param cls the 'struct GNUNET_FS_DownloadContext'
1411  * @param client handle to use for communcation with FS (we must destroy it!)
1412  */
1413 static void
1414 activate_fs_download (void *cls,
1415                       struct GNUNET_CLIENT_Connection *client)
1416 {
1417   struct GNUNET_FS_DownloadContext *dc = cls;
1418   struct GNUNET_FS_ProgressInfo pi;
1419
1420 #if DEBUG_DOWNLOAD
1421   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1422               "Download activated\n");
1423 #endif
1424   GNUNET_assert (NULL != client);
1425   GNUNET_assert (dc->client == NULL);
1426   GNUNET_assert (dc->th == NULL);
1427   dc->client = client;
1428   GNUNET_CLIENT_receive (client,
1429                          &receive_results,
1430                          dc,
1431                          GNUNET_TIME_UNIT_FOREVER_REL);
1432   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1433   GNUNET_FS_download_make_status_ (&pi, dc);
1434   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1435                                          &retry_entry,
1436                                          dc);
1437 #if DEBUG_DOWNLOAD
1438   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1439               "Asking for transmission to FS service\n");
1440 #endif
1441   dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1442                                                 sizeof (struct SearchMessage),
1443                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1444                                                 GNUNET_NO,
1445                                                 &transmit_download_request,
1446                                                 dc);
1447   GNUNET_assert (dc->th != NULL);
1448 }
1449
1450
1451 /**
1452  * We must stop to ask the FS service for our blocks.  Pause the download.
1453  *
1454  * @param cls the 'struct GNUNET_FS_DownloadContext'
1455  */
1456 static void
1457 deactivate_fs_download (void *cls)
1458 {
1459   struct GNUNET_FS_DownloadContext *dc = cls;
1460   struct GNUNET_FS_ProgressInfo pi;
1461
1462 #if DEBUG_DOWNLOAD
1463   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1464               "Download deactivated\n");
1465 #endif
1466   if (NULL != dc->th)
1467     {
1468       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1469       dc->th = NULL;
1470     }
1471   if (NULL != dc->client)
1472     {
1473       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1474       dc->client = NULL;
1475     }
1476   pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1477   GNUNET_FS_download_make_status_ (&pi, dc);
1478 }
1479
1480
1481 /**
1482  * Create SUSPEND event for the given download operation
1483  * and then clean up our state (without stop signal).
1484  *
1485  * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
1486  */
1487 void
1488 GNUNET_FS_download_signal_suspend_ (void *cls)
1489 {
1490   struct GNUNET_FS_DownloadContext *dc = cls;
1491   struct GNUNET_FS_ProgressInfo pi;
1492
1493   if (dc->top != NULL)
1494     GNUNET_FS_end_top (dc->h, dc->top);
1495   while (NULL != dc->child_head)
1496     GNUNET_FS_download_signal_suspend_ (dc->child_head);
1497   if (dc->search != NULL)
1498     {
1499       dc->search->download = NULL;
1500       dc->search = NULL;
1501     }
1502   if (dc->job_queue != NULL)
1503     {
1504       GNUNET_FS_dequeue_ (dc->job_queue);
1505       dc->job_queue = NULL;
1506     }
1507   if (dc->parent != NULL)
1508     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1509                                  dc->parent->child_tail,
1510                                  dc);
1511   pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
1512   GNUNET_FS_download_make_status_ (&pi, dc);
1513   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1514     GNUNET_SCHEDULER_cancel (dc->h->sched,
1515                              dc->task);
1516   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1517                                          &free_entry,
1518                                          NULL);
1519   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1520   GNUNET_free_non_null (dc->filename);
1521   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1522   GNUNET_FS_uri_destroy (dc->uri);
1523   GNUNET_free_non_null (dc->temp_filename);
1524   GNUNET_free_non_null (dc->serialization);
1525   GNUNET_free (dc);
1526 }
1527
1528
1529 /**
1530  * Download parts of a file.  Note that this will store
1531  * the blocks at the respective offset in the given file.  Also, the
1532  * download is still using the blocking of the underlying FS
1533  * encoding.  As a result, the download may *write* outside of the
1534  * given boundaries (if offset and length do not match the 32k FS
1535  * block boundaries). <p>
1536  *
1537  * This function should be used to focus a download towards a
1538  * particular portion of the file (optimization), not to strictly
1539  * limit the download to exactly those bytes.
1540  *
1541  * @param h handle to the file sharing subsystem
1542  * @param uri the URI of the file (determines what to download); CHK or LOC URI
1543  * @param meta known metadata for the file (can be NULL)
1544  * @param filename where to store the file, maybe NULL (then no file is
1545  *        created on disk and data must be grabbed from the callbacks)
1546  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1547  *        can be NULL (in which case we will pick a name if needed); the temporary file
1548  *        may already exist, in which case we will try to use the data that is there and
1549  *        if it is not what is desired, will overwrite it
1550  * @param offset at what offset should we start the download (typically 0)
1551  * @param length how many bytes should be downloaded starting at offset
1552  * @param anonymity anonymity level to use for the download
1553  * @param options various options
1554  * @param cctx initial value for the client context for this download
1555  * @param parent parent download to associate this download with (use NULL
1556  *        for top-level downloads; useful for manually-triggered recursive downloads)
1557  * @return context that can be used to control this download
1558  */
1559 struct GNUNET_FS_DownloadContext *
1560 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
1561                           const struct GNUNET_FS_Uri *uri,
1562                           const struct GNUNET_CONTAINER_MetaData *meta,
1563                           const char *filename,
1564                           const char *tempname,
1565                           uint64_t offset,
1566                           uint64_t length,
1567                           uint32_t anonymity,
1568                           enum GNUNET_FS_DownloadOptions options,
1569                           void *cctx,
1570                           struct GNUNET_FS_DownloadContext *parent)
1571 {
1572   struct GNUNET_FS_ProgressInfo pi;
1573   struct GNUNET_FS_DownloadContext *dc;
1574
1575   GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
1576   if ( (offset + length < offset) ||
1577        (offset + length > uri->data.chk.file_length) )
1578     {
1579       GNUNET_break (0);
1580       return NULL;
1581     }
1582 #if DEBUG_DOWNLOAD
1583   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1584               "Starting download `%s' of %llu bytes\n",
1585               filename,
1586               (unsigned long long) length);
1587 #endif
1588   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1589   dc->h = h;
1590   dc->parent = parent;
1591   if (parent != NULL)
1592     {
1593       GNUNET_CONTAINER_DLL_insert (parent->child_head,
1594                                    parent->child_tail,
1595                                    dc);
1596     }
1597   dc->uri = GNUNET_FS_uri_dup (uri);
1598   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1599   dc->client_info = cctx;
1600   dc->start_time = GNUNET_TIME_absolute_get ();
1601   if (NULL != filename)
1602     {
1603       dc->filename = GNUNET_strdup (filename);
1604       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1605         GNUNET_DISK_file_size (filename,
1606                                &dc->old_file_size,
1607                                GNUNET_YES);
1608     }
1609   if (GNUNET_FS_uri_test_loc (dc->uri))
1610     GNUNET_assert (GNUNET_OK ==
1611                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1612                                                         &dc->target));
1613   dc->offset = offset;
1614   dc->length = length;
1615   dc->anonymity = anonymity;
1616   dc->options = options;
1617   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1618   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1619   if ( (filename == NULL) &&
1620        (is_recursive_download (dc) ) )
1621     {
1622       if (tempname != NULL)
1623         dc->temp_filename = GNUNET_strdup (tempname);
1624       else
1625         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1626     }
1627
1628 #if DEBUG_DOWNLOAD
1629   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1630               "Download tree has depth %u\n",
1631               dc->treedepth);
1632 #endif
1633   if (parent == NULL)
1634     {
1635       dc->top = GNUNET_FS_make_top (dc->h,
1636                                     &GNUNET_FS_download_signal_suspend_,
1637                                     dc);
1638     }
1639   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1640   pi.value.download.specifics.start.meta = meta;
1641   GNUNET_FS_download_make_status_ (&pi, dc);
1642   schedule_block_download (dc,
1643                            &dc->uri->data.chk.chk,
1644                            0,
1645                            1 /* 0 == CHK, 1 == top */);
1646   GNUNET_FS_download_sync_ (dc);
1647   GNUNET_FS_download_start_downloading_ (dc);
1648   return dc;
1649 }
1650
1651
1652 /**
1653  * Download parts of a file based on a search result.  The download
1654  * will be associated with the search result (and the association
1655  * will be preserved when serializing/deserializing the state).
1656  * If the search is stopped, the download will not be aborted but
1657  * be 'promoted' to a stand-alone download.
1658  *
1659  * As with the other download function, this will store
1660  * the blocks at the respective offset in the given file.  Also, the
1661  * download is still using the blocking of the underlying FS
1662  * encoding.  As a result, the download may *write* outside of the
1663  * given boundaries (if offset and length do not match the 32k FS
1664  * block boundaries). <p>
1665  *
1666  * The given range can be used to focus a download towards a
1667  * particular portion of the file (optimization), not to strictly
1668  * limit the download to exactly those bytes.
1669  *
1670  * @param h handle to the file sharing subsystem
1671  * @param sr the search result to use for the download (determines uri and
1672  *        meta data and associations)
1673  * @param filename where to store the file, maybe NULL (then no file is
1674  *        created on disk and data must be grabbed from the callbacks)
1675  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1676  *        can be NULL (in which case we will pick a name if needed); the temporary file
1677  *        may already exist, in which case we will try to use the data that is there and
1678  *        if it is not what is desired, will overwrite it
1679  * @param offset at what offset should we start the download (typically 0)
1680  * @param length how many bytes should be downloaded starting at offset
1681  * @param anonymity anonymity level to use for the download
1682  * @param options various download options
1683  * @param cctx initial value for the client context for this download
1684  * @return context that can be used to control this download
1685  */
1686 struct GNUNET_FS_DownloadContext *
1687 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
1688                                       struct GNUNET_FS_SearchResult *sr,
1689                                       const char *filename,
1690                                       const char *tempname,
1691                                       uint64_t offset,
1692                                       uint64_t length,
1693                                       uint32_t anonymity,
1694                                       enum GNUNET_FS_DownloadOptions options,
1695                                       void *cctx)
1696 {
1697   struct GNUNET_FS_ProgressInfo pi;
1698   struct GNUNET_FS_DownloadContext *dc;
1699
1700   if ( (sr == NULL) ||
1701        (sr->download != NULL) )
1702     {
1703       GNUNET_break (0);
1704       return NULL;
1705     }
1706   GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri));
1707   if ( (offset + length < offset) ||
1708        (offset + length > sr->uri->data.chk.file_length) )
1709     {
1710       GNUNET_break (0);
1711       return NULL;
1712     }
1713 #if DEBUG_DOWNLOAD
1714   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1715               "Starting download `%s' of %llu bytes\n",
1716               filename,
1717               (unsigned long long) length);
1718 #endif
1719   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1720   dc->h = h;
1721   dc->search = sr;
1722   sr->download = dc;
1723   if (sr->probe_ctx != NULL)
1724     {
1725       GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
1726       sr->probe_ctx = NULL;
1727     }
1728   dc->uri = GNUNET_FS_uri_dup (sr->uri);
1729   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
1730   dc->client_info = cctx;
1731   dc->start_time = GNUNET_TIME_absolute_get ();
1732   if (NULL != filename)
1733     {
1734       dc->filename = GNUNET_strdup (filename);
1735       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1736         GNUNET_DISK_file_size (filename,
1737                                &dc->old_file_size,
1738                                GNUNET_YES);
1739     }
1740   if (GNUNET_FS_uri_test_loc (dc->uri))
1741     GNUNET_assert (GNUNET_OK ==
1742                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1743                                                         &dc->target));
1744   dc->offset = offset;
1745   dc->length = length;
1746   dc->anonymity = anonymity;
1747   dc->options = options;
1748   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1749   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1750   if ( (filename == NULL) &&
1751        (is_recursive_download (dc) ) )
1752     {
1753       if (tempname != NULL)
1754         dc->temp_filename = GNUNET_strdup (tempname);
1755       else
1756         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1757     }
1758
1759 #if DEBUG_DOWNLOAD
1760   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1761               "Download tree has depth %u\n",
1762               dc->treedepth);
1763 #endif
1764   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1765   pi.value.download.specifics.start.meta = dc->meta;
1766   GNUNET_FS_download_make_status_ (&pi, dc);
1767   schedule_block_download (dc,
1768                            &dc->uri->data.chk.chk,
1769                            0,
1770                            1 /* 0 == CHK, 1 == top */);
1771   GNUNET_FS_download_sync_ (dc);
1772   GNUNET_FS_download_start_downloading_ (dc);
1773   return dc;
1774 }
1775
1776
1777 /**
1778  * Start the downloading process (by entering the queue).
1779  *
1780  * @param dc our download context
1781  */
1782 void
1783 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
1784 {
1785   GNUNET_assert (dc->job_queue == NULL);
1786   dc->job_queue = GNUNET_FS_queue_ (dc->h,
1787                                     &activate_fs_download,
1788                                     &deactivate_fs_download,
1789                                     dc,
1790                                     (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
1791 }
1792
1793
1794 /**
1795  * Stop a download (aborts if download is incomplete).
1796  *
1797  * @param dc handle for the download
1798  * @param do_delete delete files of incomplete downloads
1799  */
1800 void
1801 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
1802                          int do_delete)
1803 {
1804   struct GNUNET_FS_ProgressInfo pi;
1805   int have_children;
1806
1807   if (dc->top != NULL)
1808     GNUNET_FS_end_top (dc->h, dc->top);
1809   if (dc->search != NULL)
1810     {
1811       dc->search->download = NULL;
1812       dc->search = NULL;
1813     }
1814   if (dc->job_queue != NULL)
1815     {
1816       GNUNET_FS_dequeue_ (dc->job_queue);
1817       dc->job_queue = NULL;
1818     }
1819   have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
1820   while (NULL != dc->child_head)
1821     GNUNET_FS_download_stop (dc->child_head,
1822                              do_delete);
1823   if (dc->parent != NULL)
1824     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1825                                  dc->parent->child_tail,
1826                                  dc);
1827   if (dc->serialization != NULL)
1828     GNUNET_FS_remove_sync_file_ (dc->h,
1829                                  ( (dc->parent != NULL)  || (dc->search != NULL) )
1830                                  ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1831                                  : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
1832                                  dc->serialization);
1833   if ( (GNUNET_YES == have_children) &&
1834        (dc->parent == NULL) )
1835     GNUNET_FS_remove_sync_dir_ (dc->h,
1836                                 (dc->search != NULL)
1837                                 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1838                                 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
1839                                 dc->serialization);
1840   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
1841   GNUNET_FS_download_make_status_ (&pi, dc);
1842   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1843     GNUNET_SCHEDULER_cancel (dc->h->sched,
1844                              dc->task);
1845   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1846                                          &free_entry,
1847                                          NULL);
1848   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1849   if (dc->filename != NULL)
1850     {
1851       if ( (dc->completed != dc->length) &&
1852            (GNUNET_YES == do_delete) )
1853         {
1854           if (0 != UNLINK (dc->filename))
1855             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1856                                       "unlink",
1857                                       dc->filename);
1858         }
1859       GNUNET_free (dc->filename);
1860     }
1861   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1862   GNUNET_FS_uri_destroy (dc->uri);
1863   if (NULL != dc->temp_filename)
1864     {
1865       if (0 != UNLINK (dc->temp_filename))
1866         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1867                                   "unlink",
1868                                   dc->temp_filename);
1869       GNUNET_free (dc->temp_filename);
1870     }
1871   GNUNET_free_non_null (dc->serialization);
1872   GNUNET_free (dc);
1873 }
1874
1875 /* end of fs_download.c */