src/fs/fs_download.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file fs/fs_download.c
  22  * @brief download methods
  23  * @author Christian Grothoff
  24  *
  25  * TODO:
  26  * - location URI suppport (can wait, easy)
  27  * - different priority for scheduling probe downloads?
  28  * - check if iblocks can be computed from existing blocks (can wait, hard)
  29  */
  30 #include "platform.h"
  31 #include "gnunet_constants.h"
  32 #include "gnunet_fs_service.h"
  33 #include "fs.h"
  34 #include "fs_tree.h"
  35
  36 #define DEBUG_DOWNLOAD GNUNET_NO
  37
  38 /**
  39  * Determine if the given download (options and meta data) should cause
  40  * use to try to do a recursive download.
  41  */
  42 static int
  43 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
  44 {
  45   return  (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
  46     ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
  47       ( (dc->meta == NULL) &&
  48         ( (NULL == dc->filename) ||
  49           ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
  50             (NULL !=
  51              strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
  52                      GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
  53 }
  54
  55
  56 /**
  57  * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
  58  * only have to truncate the file once we're done).
  59  *
  60  * Given the offset of a block (with respect to the DBLOCKS) and its
  61  * depth, return the offset where we would store this block in the
  62  * file.
  63  *
  64  * @param fsize overall file size
  65  * @param off offset of the block in the file
  66  * @param depth depth of the block in the tree
  67  * @param treedepth maximum depth of the tree
  68  * @return off for DBLOCKS (depth == treedepth),
  69  *         otherwise an offset past the end
  70  *         of the file that does not overlap
  71  *         with the range for any other block
  72  */
  73 static uint64_t
  74 compute_disk_offset (uint64_t fsize,
  75                      uint64_t off,
  76                      unsigned int depth,
  77                      unsigned int treedepth)
  78 {
  79   unsigned int i;
  80   uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
  81   uint64_t loff; /* where do IBlocks for depth "i" start? */
  82   unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
  83
  84   if (depth == treedepth)
  85     return off;
  86   /* first IBlocks start at the end of file, rounded up
  87      to full DBLOCK_SIZE */
  88   loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
  89   lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
  90   GNUNET_assert (0 == (off % DBLOCK_SIZE));
  91   ioff = (off / DBLOCK_SIZE);
  92   for (i=treedepth-1;i>depth;i--)
  93     {
  94       loff += lsize;
  95       lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
  96       GNUNET_assert (lsize > 0);
  97       GNUNET_assert (0 == (ioff % CHK_PER_INODE));
  98       ioff /= CHK_PER_INODE;
  99     }
 100   return loff + ioff * sizeof (struct ContentHashKey);
 101 }
 102
 103
 104 /**
 105  * Given a file of the specified treedepth and a block at the given
 106  * offset and depth, calculate the offset for the CHK at the given
 107  * index.
 108  *
 109  * @param offset the offset of the first
 110  *        DBLOCK in the subtree of the
 111  *        identified IBLOCK
 112  * @param depth the depth of the IBLOCK in the tree
 113  * @param treedepth overall depth of the tree
 114  * @param k which CHK in the IBLOCK are we
 115  *        talking about
 116  * @return offset if k=0, otherwise an appropriately
 117  *         larger value (i.e., if depth = treedepth-1,
 118  *         the returned value should be offset+DBLOCK_SIZE)
 119  */
 120 static uint64_t
 121 compute_dblock_offset (uint64_t offset,
 122                        unsigned int depth,
 123                        unsigned int treedepth,
 124                        unsigned int k)
 125 {
 126   unsigned int i;
 127   uint64_t lsize; /* what is the size of the sum of all DBlocks
 128                      that a CHK at depth i corresponds to? */
 129
 130   if (depth == treedepth)
 131     return offset;
 132   lsize = DBLOCK_SIZE;
 133   for (i=treedepth-1;i>depth;i--)
 134     lsize *= CHK_PER_INODE;
 135   return offset + k * lsize;
 136 }
 137
 138
 139 /**
 140  * Fill in all of the generic fields for a download event and call the
 141  * callback.
 142  *
 143  * @param pi structure to fill in
 144  * @param dc overall download context
 145  */
 146 void
 147 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
 148                                  struct GNUNET_FS_DownloadContext *dc)
 149 {
 150   pi->value.download.dc = dc;
 151   pi->value.download.cctx
 152     = dc->client_info;
 153   pi->value.download.pctx
 154     = (dc->parent == NULL) ? NULL : dc->parent->client_info;
 155   pi->value.download.sctx
 156     = (dc->search == NULL) ? NULL : dc->search->client_info;
 157   pi->value.download.uri
 158     = dc->uri;
 159   pi->value.download.filename
 160     = dc->filename;
 161   pi->value.download.size
 162     = dc->length;
 163   pi->value.download.duration
 164     = GNUNET_TIME_absolute_get_duration (dc->start_time);
 165   pi->value.download.completed
 166     = dc->completed;
 167   pi->value.download.anonymity
 168     = dc->anonymity;
 169   pi->value.download.eta
 170     = GNUNET_TIME_calculate_eta (dc->start_time,
 171                                  dc->completed,
 172                                  dc->length);
 173   pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
 174   if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
 175     dc->client_info = dc->h->upcb (dc->h->upcb_cls,
 176                                    pi);
 177   else
 178     dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
 179                                                         pi);
 180 }
 181
 182 /**
 183  * We're ready to transmit a search request to the
 184  * file-sharing service.  Do it.  If there is
 185  * more than one request pending, try to send
 186  * multiple or request another transmission.
 187  *
 188  * @param cls closure
 189  * @param size number of bytes available in buf
 190  * @param buf where the callee should write the message
 191  * @return number of bytes written to buf
 192  */
 193 static size_t
 194 transmit_download_request (void *cls,
 195                            size_t size,
 196                            void *buf);
 197
 198
 199 /**
 200  * Closure for iterator processing results.
 201  */
 202 struct ProcessResultClosure
 203 {
 204
 205   /**
 206    * Hash of data.
 207    */
 208   GNUNET_HashCode query;
 209
 210   /**
 211    * Data found in P2P network.
 212    */
 213   const void *data;
 214
 215   /**
 216    * Our download context.
 217    */
 218   struct GNUNET_FS_DownloadContext *dc;
 219
 220   /**
 221    * Number of bytes in data.
 222    */
 223   size_t size;
 224
 225   /**
 226    * Type of data.
 227    */
 228   enum GNUNET_BLOCK_Type type;
 229
 230   /**
 231    * Flag to indicate if this block should be stored on disk.
 232    */
 233   int do_store;
 234
 235 };
 236
 237
 238 /**
 239  * Iterator over entries in the pending requests in the 'active' map for the
 240  * reply that we just got.
 241  *
 242  * @param cls closure (our 'struct ProcessResultClosure')
 243  * @param key query for the given value / request
 244  * @param value value in the hash map (a 'struct DownloadRequest')
 245  * @return GNUNET_YES (we should continue to iterate); unless serious error
 246  */
 247 static int
 248 process_result_with_request (void *cls,
 249                              const GNUNET_HashCode * key,
 250                              void *value);
 251
 252
 253
 254 /**
 255  * Schedule the download of the specified block in the tree.
 256  *
 257  * @param dc overall download this block belongs to
 258  * @param chk content-hash-key of the block
 259  * @param offset offset of the block in the file
 260  *         (for IBlocks, the offset is the lowest
 261  *          offset of any DBlock in the subtree under
 262  *          the IBlock)
 263  * @param depth depth of the block, 0 is the root of the tree
 264  */
 265 static void
 266 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
 267                          const struct ContentHashKey *chk,
 268                          uint64_t offset,
 269                          unsigned int depth)
 270 {
 271   struct DownloadRequest *sm;
 272   uint64_t total;
 273   uint64_t off;
 274   size_t len;
 275   char block[DBLOCK_SIZE];
 276   GNUNET_HashCode key;
 277   struct ProcessResultClosure prc;
 278   struct GNUNET_DISK_FileHandle *fh;
 279
 280 #if DEBUG_DOWNLOAD
 281   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 282               "Scheduling download at offset %llu and depth %u for `%s'\n",
 283               (unsigned long long) offset,
 284               depth,
 285               GNUNET_h2s (&chk->query));
 286 #endif
 287   total = GNUNET_ntohll (dc->uri->data.chk.file_length);
 288   off = compute_disk_offset (total,
 289                              offset,
 290                              depth,
 291                              dc->treedepth);
 292   len = GNUNET_FS_tree_calculate_block_size (total,
 293                                              dc->treedepth,
 294                                              offset,
 295                                              depth);
 296   sm = GNUNET_malloc (sizeof (struct DownloadRequest));
 297   sm->chk = *chk;
 298   sm->offset = offset;
 299   sm->depth = depth;
 300   sm->is_pending = GNUNET_YES;
 301   sm->next = dc->pending;
 302   dc->pending = sm;
 303   GNUNET_CONTAINER_multihashmap_put (dc->active,
 304                                      &chk->query,
 305                                      sm,
 306                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
 307   fh = NULL;
 308   if ( (dc->old_file_size > off) &&
 309        (dc->filename != NULL) )
 310     fh = GNUNET_DISK_file_open (dc->filename,
 311                                 GNUNET_DISK_OPEN_READ,
 312                                 GNUNET_DISK_PERM_NONE);
 313   if ( (fh != NULL) &&
 314        (off  ==
 315         GNUNET_DISK_file_seek (fh,
 316                                off,
 317                                GNUNET_DISK_SEEK_SET) ) &&
 318        (len ==
 319         GNUNET_DISK_file_read (fh,
 320                                block,
 321                                len)) )
 322     {
 323       GNUNET_CRYPTO_hash (block, len, &key);
 324       if (0 == memcmp (&key,
 325                        &chk->key,
 326                        sizeof (GNUNET_HashCode)))
 327         {
 328           char enc[len];
 329           struct GNUNET_CRYPTO_AesSessionKey sk;
 330           struct GNUNET_CRYPTO_AesInitializationVector iv;
 331           GNUNET_HashCode query;
 332
 333           GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
 334           if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
 335                                                &sk,
 336                                                &iv,
 337                                                enc))
 338             {
 339               GNUNET_break (0);
 340               goto do_download;
 341             }
 342           GNUNET_CRYPTO_hash (enc, len, &query);
 343           if (0 == memcmp (&query,
 344                            &chk->query,
 345                            sizeof (GNUNET_HashCode)))
 346             {
 347 #if DEBUG_DOWNLOAD
 348               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 349                           "Matching block already present, no need for download!\n");
 350 #endif
 351               /* already got it! */
 352               prc.dc = dc;
 353               prc.data = enc;
 354               prc.size = len;
 355               prc.type = (dc->treedepth == depth)
 356                 ? GNUNET_BLOCK_TYPE_DBLOCK
 357                 : GNUNET_BLOCK_TYPE_IBLOCK;
 358               prc.query = chk->query;
 359               prc.do_store = GNUNET_NO; /* useless */
 360               process_result_with_request (&prc,
 361                                            &key,
 362                                            sm);
 363             }
 364           else
 365             {
 366               GNUNET_break_op (0);
 367             }
 368           GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 369           return;
 370         }
 371     }
 372  do_download:
 373   if (fh != NULL)
 374     GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 375   if (depth < dc->treedepth)
 376     {
 377       // FIXME: try if we could
 378       // reconstitute this IBLOCK
 379       // from the existing blocks on disk (can wait)
 380       // (read block(s), encode, compare with
 381       // query; if matches, simply return)
 382     }
 383
 384   if ( (dc->th == NULL) &&
 385        (dc->client != NULL) )
 386     {
 387 #if DEBUG_DOWNLOAD
 388       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 389                   "Asking for transmission to FS service\n");
 390 #endif
 391       dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
 392                                                     sizeof (struct SearchMessage),
 393                                                     GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 394                                                     GNUNET_NO,
 395                                                     &transmit_download_request,
 396                                                     dc);
 397     }
 398   else
 399     {
 400 #if DEBUG_DOWNLOAD
 401       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 402                   "Transmission request not issued (%p %p)\n",
 403                   dc->th,
 404                   dc->client);
 405 #endif
 406
 407     }
 408
 409 }
 410
 411
 412
 413 /**
 414  * Suggest a filename based on given metadata.
 415  *
 416  * @param md given meta data
 417  * @return NULL if meta data is useless for suggesting a filename
 418  */
 419 char *
 420 GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md)
 421 {
 422   static const char *mimeMap[][2] = {
 423     {"application/bz2", ".bz2"},
 424     {"application/gnunet-directory", ".gnd"},
 425     {"application/java", ".class"},
 426     {"application/msword", ".doc"},
 427     {"application/ogg", ".ogg"},
 428     {"application/pdf", ".pdf"},
 429     {"application/pgp-keys", ".key"},
 430     {"application/pgp-signature", ".pgp"},
 431     {"application/postscript", ".ps"},
 432     {"application/rar", ".rar"},
 433     {"application/rtf", ".rtf"},
 434     {"application/xml", ".xml"},
 435     {"application/x-debian-package", ".deb"},
 436     {"application/x-dvi", ".dvi"},
 437     {"applixation/x-flac", ".flac"},
 438     {"applixation/x-gzip", ".gz"},
 439     {"application/x-java-archive", ".jar"},
 440     {"application/x-java-vm", ".class"},
 441     {"application/x-python-code", ".pyc"},
 442     {"application/x-redhat-package-manager", ".rpm"},
 443     {"application/x-rpm", ".rpm"},
 444     {"application/x-tar", ".tar"},
 445     {"application/x-tex-pk", ".pk"},
 446     {"application/x-texinfo", ".texinfo"},
 447     {"application/x-xcf", ".xcf"},
 448     {"application/x-xfig", ".xfig"},
 449     {"application/zip", ".zip"},
 450
 451     {"audio/midi", ".midi"},
 452     {"audio/mpeg", ".mp3"},
 453     {"audio/real", ".rm"},
 454     {"audio/x-wav", ".wav"},
 455
 456     {"image/gif", ".gif"},
 457     {"image/jpeg", ".jpg"},
 458     {"image/pcx", ".pcx"},
 459     {"image/png", ".png"},
 460     {"image/tiff", ".tiff"},
 461     {"image/x-ms-bmp", ".bmp"},
 462     {"image/x-xpixmap", ".xpm"},
 463
 464     {"text/css", ".css"},
 465     {"text/html", ".html"},
 466     {"text/plain", ".txt"},
 467     {"text/rtf", ".rtf"},
 468     {"text/x-c++hdr", ".h++"},
 469     {"text/x-c++src", ".c++"},
 470     {"text/x-chdr", ".h"},
 471     {"text/x-csrc", ".c"},
 472     {"text/x-java", ".java"},
 473     {"text/x-moc", ".moc"},
 474     {"text/x-pascal", ".pas"},
 475     {"text/x-perl", ".pl"},
 476     {"text/x-python", ".py"},
 477     {"text/x-tex", ".tex"},
 478
 479     {"video/avi", ".avi"},
 480     {"video/mpeg", ".mpeg"},
 481     {"video/quicktime", ".qt"},
 482     {"video/real", ".rm"},
 483     {"video/x-msvideo", ".avi"},
 484     {NULL, NULL},
 485   };
 486   char *ret;
 487   unsigned int i;
 488   char *mime;
 489   char *base;
 490   const char *ext;
 491
 492   ret = GNUNET_CONTAINER_meta_data_get_by_type (md,
 493                                                 EXTRACTOR_METATYPE_FILENAME);
 494   if (ret != NULL)
 495     return ret;
 496   ext = NULL;
 497   mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
 498                                                  EXTRACTOR_METATYPE_MIMETYPE);
 499   if (mime != NULL)
 500     {
 501       i = 0;
 502       while ( (mimeMap[i][0] != NULL) &&
 503               (0 != strcmp (mime, mimeMap[i][0])))
 504         i++;
 505       if (mimeMap[i][1] == NULL)
 506         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG |
 507                     GNUNET_ERROR_TYPE_BULK,
 508                     _("Did not find mime type `%s' in extension list.\n"),
 509                     mime);
 510       else
 511         ext = mimeMap[i][1];
 512       GNUNET_free (mime);
 513     }
 514   base = GNUNET_CONTAINER_meta_data_get_first_by_types (md,
 515                                                         EXTRACTOR_METATYPE_TITLE,
 516                                                         EXTRACTOR_METATYPE_BOOK_TITLE,
 517                                                         EXTRACTOR_METATYPE_ORIGINAL_TITLE,
 518                                                         EXTRACTOR_METATYPE_PACKAGE_NAME,
 519                                                         EXTRACTOR_METATYPE_URL,
 520                                                         EXTRACTOR_METATYPE_URI,
 521                                                         EXTRACTOR_METATYPE_DESCRIPTION,
 522                                                         EXTRACTOR_METATYPE_ISRC,
 523                                                         EXTRACTOR_METATYPE_JOURNAL_NAME,
 524                                                         EXTRACTOR_METATYPE_AUTHOR_NAME,
 525                                                         EXTRACTOR_METATYPE_SUBJECT,
 526                                                         EXTRACTOR_METATYPE_ALBUM,
 527                                                         EXTRACTOR_METATYPE_ARTIST,
 528                                                         EXTRACTOR_METATYPE_KEYWORDS,
 529                                                         EXTRACTOR_METATYPE_COMMENT,
 530                                                         EXTRACTOR_METATYPE_UNKNOWN,
 531                                                         -1);
 532   if ( (base == NULL) &&
 533        (ext == NULL) )
 534     return NULL;
 535   if (base == NULL)
 536     return GNUNET_strdup (ext);
 537   if (ext == NULL)
 538     return base;
 539   GNUNET_asprintf (&ret,
 540                    "%s%s",
 541                    base,
 542                    ext);
 543   GNUNET_free (base);
 544   return ret;
 545 }
 546
 547
 548 /**
 549  * We've lost our connection with the FS service.
 550  * Re-establish it and re-transmit all of our
 551  * pending requests.
 552  *
 553  * @param dc download context that is having trouble
 554  */
 555 static void
 556 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 557
 558
 559 /**
 560  * We found an entry in a directory.  Check if the respective child
 561  * already exists and if not create the respective child download.
 562  *
 563  * @param cls the parent download
 564  * @param filename name of the file in the directory
 565  * @param uri URI of the file (CHK or LOC)
 566  * @param meta meta data of the file
 567  * @param length number of bytes in data
 568  * @param data contents of the file (or NULL if they were not inlined)
 569  */
 570 static void
 571 trigger_recursive_download (void *cls,
 572                             const char *filename,
 573                             const struct GNUNET_FS_Uri *uri,
 574                             const struct GNUNET_CONTAINER_MetaData *meta,
 575                             size_t length,
 576                             const void *data);
 577
 578
 579 /**
 580  * We're done downloading a directory.  Open the file and
 581  * trigger all of the (remaining) child downloads.
 582  *
 583  * @param dc context of download that just completed
 584  */
 585 static void
 586 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 587 {
 588   size_t size;
 589   uint64_t size64;
 590   void *data;
 591   struct GNUNET_DISK_FileHandle *h;
 592   struct GNUNET_DISK_MapHandle *m;
 593
 594   size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
 595   size = (size_t) size64;
 596   if (size64 != (uint64_t) size)
 597     {
 598       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 599                   _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
 600       return;
 601     }
 602   if (dc->filename != NULL)
 603     {
 604       h = GNUNET_DISK_file_open (dc->filename,
 605                                  GNUNET_DISK_OPEN_READ,
 606                                  GNUNET_DISK_PERM_NONE);
 607     }
 608   else
 609     {
 610       GNUNET_assert (dc->temp_filename != NULL);
 611       h = GNUNET_DISK_file_open (dc->temp_filename,
 612                                  GNUNET_DISK_OPEN_READ,
 613                                  GNUNET_DISK_PERM_NONE);
 614     }
 615   if (h == NULL)
 616     return; /* oops */
 617   data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
 618   if (data == NULL)
 619     {
 620       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 621                   _("Directory too large for system address space\n"));
 622     }
 623   else
 624     {
 625       GNUNET_FS_directory_list_contents (size,
 626                                          data,
 627                                          0,
 628                                          &trigger_recursive_download,
 629                                          dc);
 630       GNUNET_DISK_file_unmap (m);
 631     }
 632   GNUNET_DISK_file_close (h);
 633   if (dc->filename == NULL)
 634     {
 635       if (0 != UNLINK (dc->temp_filename))
 636         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
 637                                   "unlink",
 638                                   dc->temp_filename);
 639       GNUNET_free (dc->temp_filename);
 640       dc->temp_filename = NULL;
 641     }
 642 }
 643
 644
 645 /**
 646  * Check if all child-downloads have completed and
 647  * if so, signal completion (and possibly recurse to
 648  * parent).
 649  */
 650 static void
 651 check_completed (struct GNUNET_FS_DownloadContext *dc)
 652 {
 653   struct GNUNET_FS_ProgressInfo pi;
 654   struct GNUNET_FS_DownloadContext *pos;
 655
 656   pos = dc->child_head;
 657   while (pos != NULL)
 658     {
 659       if ( (pos->emsg == NULL) &&
 660            (pos->completed < pos->length) )
 661         return; /* not done yet */
 662       if ( (pos->child_head != NULL) &&
 663            (pos->has_finished != GNUNET_YES) )
 664         return; /* not transitively done yet */
 665       pos = pos->next;
 666     }
 667   dc->has_finished = GNUNET_YES;
 668   GNUNET_FS_download_sync_ (dc);
 669   /* signal completion */
 670   pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
 671   GNUNET_FS_download_make_status_ (&pi, dc);
 672   if (dc->parent != NULL)
 673     check_completed (dc->parent);
 674 }
 675
 676
 677 /**
 678  * We found an entry in a directory.  Check if the respective child
 679  * already exists and if not create the respective child download.
 680  *
 681  * @param cls the parent download
 682  * @param filename name of the file in the directory
 683  * @param uri URI of the file (CHK or LOC)
 684  * @param meta meta data of the file
 685  * @param length number of bytes in data
 686  * @param data contents of the file (or NULL if they were not inlined)
 687  */
 688 static void
 689 trigger_recursive_download (void *cls,
 690                             const char *filename,
 691                             const struct GNUNET_FS_Uri *uri,
 692                             const struct GNUNET_CONTAINER_MetaData *meta,
 693                             size_t length,
 694                             const void *data)
 695 {
 696   struct GNUNET_FS_DownloadContext *dc = cls;
 697   struct GNUNET_FS_DownloadContext *cpos;
 698   struct GNUNET_DISK_FileHandle *fh;
 699   char *temp_name;
 700   const char *real_name;
 701   char *fn;
 702   char *us;
 703   char *ext;
 704   char *dn;
 705   char *full_name;
 706
 707   if (NULL == uri)
 708     return; /* entry for the directory itself */
 709   cpos = dc->child_head;
 710   while (cpos != NULL)
 711     {
 712       if ( (GNUNET_FS_uri_test_equal (uri,
 713                                       cpos->uri)) ||
 714            ( (filename != NULL) &&
 715              (0 == strcmp (cpos->filename,
 716                            filename)) ) )
 717         break;
 718       cpos = cpos->next;
 719     }
 720   if (cpos != NULL)
 721     return; /* already exists */
 722   fn = NULL;
 723   if (NULL == filename)
 724     {
 725       fn = GNUNET_FS_meta_data_suggest_filename (meta);
 726       if (fn == NULL)
 727         {
 728           us = GNUNET_FS_uri_to_string (uri);
 729           fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_PREFIX
 730                                            GNUNET_FS_URI_CHK_INFIX)]);
 731           GNUNET_free (us);
 732         }
 733       else if (fn[0] == '.')
 734         {
 735           ext = fn;
 736           us = GNUNET_FS_uri_to_string (uri);
 737           GNUNET_asprintf (&fn,
 738                            "%s%s",
 739                            &us[strlen (GNUNET_FS_URI_PREFIX
 740                                        GNUNET_FS_URI_CHK_INFIX)], ext);
 741           GNUNET_free (ext);
 742           GNUNET_free (us);
 743         }
 744       filename = fn;
 745     }
 746   if (dc->filename == NULL)
 747     {
 748       full_name = NULL;
 749     }
 750   else
 751     {
 752       dn = GNUNET_strdup (dc->filename);
 753       GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 754                      (NULL !=
 755                       strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 756                               GNUNET_FS_DIRECTORY_EXT)) );
 757       if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 758            (NULL !=
 759             strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 760                     GNUNET_FS_DIRECTORY_EXT)) )
 761         dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
 762       if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
 763            ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
 764              (NULL ==
 765               strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
 766                       GNUNET_FS_DIRECTORY_EXT)) ) )
 767         {
 768           GNUNET_asprintf (&full_name,
 769                            "%s%s%s%s",
 770                            dn,
 771                            DIR_SEPARATOR_STR,
 772                            filename,
 773                            GNUNET_FS_DIRECTORY_EXT);
 774         }
 775       else
 776         {
 777           GNUNET_asprintf (&full_name,
 778                            "%s%s%s",
 779                            dn,
 780                            DIR_SEPARATOR_STR,
 781                            filename);
 782         }
 783       GNUNET_free (dn);
 784     }
 785   if ( (full_name != NULL) &&
 786        (GNUNET_OK !=
 787         GNUNET_DISK_directory_create_for_file (full_name)) )
 788     {
 789       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 790                   _("Failed to create directory for recursive download of `%s'\n"),
 791                   full_name);
 792       GNUNET_free (full_name);
 793       GNUNET_free_non_null (fn);
 794       return;
 795     }
 796
 797   temp_name = NULL;
 798   if ( (data != NULL) &&
 799        (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
 800     {
 801       if (full_name == NULL)
 802         {
 803           temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
 804           real_name = temp_name;
 805         }
 806       else
 807         {
 808           real_name = full_name;
 809         }
 810       /* write to disk, then trigger normal download which will instantly progress to completion */
 811       fh = GNUNET_DISK_file_open (real_name,
 812                                   GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
 813                                   GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
 814       if (fh == NULL)
 815         {
 816           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 817                                     "open",
 818                                     real_name);
 819           GNUNET_free (full_name);
 820           GNUNET_free_non_null (fn);
 821           return;
 822         }
 823       if (length !=
 824           GNUNET_DISK_file_write (fh,
 825                                   data,
 826                                   length))
 827         {
 828           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 829                                     "write",
 830                                     full_name);
 831         }
 832       GNUNET_DISK_file_close (fh);
 833     }
 834   GNUNET_FS_download_start (dc->h,
 835                             uri,
 836                             meta,
 837                             full_name, temp_name,
 838                             0,
 839                             GNUNET_FS_uri_chk_get_file_size (uri),
 840                             dc->anonymity,
 841                             dc->options,
 842                             NULL,
 843                             dc);
 844   GNUNET_free_non_null (full_name);
 845   GNUNET_free_non_null (temp_name);
 846   GNUNET_free_non_null (fn);
 847 }
 848
 849
 850 /**
 851  * Free entries in the map.
 852  *
 853  * @param cls unused (NULL)
 854  * @param key unused
 855  * @param entry entry of type "struct DownloadRequest" which is freed
 856  * @return GNUNET_OK
 857  */
 858 static int
 859 free_entry (void *cls,
 860             const GNUNET_HashCode *key,
 861             void *entry)
 862 {
 863   GNUNET_free (entry);
 864   return GNUNET_OK;
 865 }
 866
 867
 868 /**
 869  * Iterator over entries in the pending requests in the 'active' map for the
 870  * reply that we just got.
 871  *
 872  * @param cls closure (our 'struct ProcessResultClosure')
 873  * @param key query for the given value / request
 874  * @param value value in the hash map (a 'struct DownloadRequest')
 875  * @return GNUNET_YES (we should continue to iterate); unless serious error
 876  */
 877 static int
 878 process_result_with_request (void *cls,
 879                              const GNUNET_HashCode * key,
 880                              void *value)
 881 {
 882   struct ProcessResultClosure *prc = cls;
 883   struct DownloadRequest *sm = value;
 884   struct DownloadRequest *ppos;
 885   struct DownloadRequest *pprev;
 886   struct GNUNET_DISK_FileHandle *fh;
 887   struct GNUNET_FS_DownloadContext *dc = prc->dc;
 888   struct GNUNET_CRYPTO_AesSessionKey skey;
 889   struct GNUNET_CRYPTO_AesInitializationVector iv;
 890   char pt[prc->size];
 891   struct GNUNET_FS_ProgressInfo pi;
 892   uint64_t off;
 893   size_t bs;
 894   size_t app;
 895   int i;
 896   struct ContentHashKey *chk;
 897
 898   fh = NULL;
 899   bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
 900                                             dc->treedepth,
 901                                             sm->offset,
 902                                             sm->depth);
 903   if (prc->size != bs)
 904     {
 905 #if DEBUG_DOWNLOAD
 906       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 907                   "Internal error or bogus download URI (expected %u bytes, got %u)\n",
 908                   bs,
 909                   prc->size);
 910 #endif
 911       dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
 912       goto signal_error;
 913     }
 914   GNUNET_assert (GNUNET_YES ==
 915                  GNUNET_CONTAINER_multihashmap_remove (dc->active,
 916                                                        &prc->query,
 917                                                        sm));
 918   /* if this request is on the pending list, remove it! */
 919   pprev = NULL;
 920   ppos = dc->pending;
 921   while (ppos != NULL)
 922     {
 923       if (ppos == sm)
 924         {
 925           if (pprev == NULL)
 926             dc->pending = ppos->next;
 927           else
 928             pprev->next = ppos->next;
 929           break;
 930         }
 931       pprev = ppos;
 932       ppos = ppos->next;
 933     }
 934   GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
 935   if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
 936                                        prc->size,
 937                                        &skey,
 938                                        &iv,
 939                                        pt))
 940     {
 941       GNUNET_break (0);
 942       dc->emsg = GNUNET_strdup ("internal error decrypting content");
 943       goto signal_error;
 944     }
 945   off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
 946                              sm->offset,
 947                              sm->depth,
 948                              dc->treedepth);
 949   /* save to disk */
 950   if ( ( GNUNET_YES == prc->do_store) &&
 951        ( (dc->filename != NULL) ||
 952          (is_recursive_download (dc)) ) &&
 953        ( (sm->depth == dc->treedepth) ||
 954          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 955     {
 956       fh = GNUNET_DISK_file_open (dc->filename != NULL
 957                                   ? dc->filename
 958                                   : dc->temp_filename,
 959                                   GNUNET_DISK_OPEN_READWRITE |
 960                                   GNUNET_DISK_OPEN_CREATE,
 961                                   GNUNET_DISK_PERM_USER_READ |
 962                                   GNUNET_DISK_PERM_USER_WRITE |
 963                                   GNUNET_DISK_PERM_GROUP_READ |
 964                                   GNUNET_DISK_PERM_OTHER_READ);
 965     }
 966   if ( (NULL == fh) &&
 967        (GNUNET_YES == prc->do_store) &&
 968        ( (dc->filename != NULL) ||
 969          (is_recursive_download (dc)) ) &&
 970        ( (sm->depth == dc->treedepth) ||
 971          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 972     {
 973       GNUNET_asprintf (&dc->emsg,
 974                        _("Download failed: could not open file `%s': %s\n"),
 975                        dc->filename,
 976                        STRERROR (errno));
 977       goto signal_error;
 978     }
 979   if (fh != NULL)
 980     {
 981 #if DEBUG_DOWNLOAD
 982       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 983                   "Saving decrypted block to disk at offset %llu\n",
 984                   (unsigned long long) off);
 985 #endif
 986       if ( (off  !=
 987             GNUNET_DISK_file_seek (fh,
 988                                    off,
 989                                    GNUNET_DISK_SEEK_SET) ) )
 990         {
 991           GNUNET_asprintf (&dc->emsg,
 992                            _("Failed to seek to offset %llu in file `%s': %s\n"),
 993                            (unsigned long long) off,
 994                            dc->filename,
 995                            STRERROR (errno));
 996           goto signal_error;
 997         }
 998       if (prc->size !=
 999           GNUNET_DISK_file_write (fh,
1000                                   pt,
1001                                   prc->size))
1002         {
1003           GNUNET_asprintf (&dc->emsg,
1004                            _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1005                            (unsigned int) prc->size,
1006                            (unsigned long long) off,
1007                            dc->filename,
1008                            STRERROR (errno));
1009           goto signal_error;
1010         }
1011       GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1012       fh = NULL;
1013     }
1014   if (sm->depth == dc->treedepth)
1015     {
1016       app = prc->size;
1017       if (sm->offset < dc->offset)
1018         {
1019           /* starting offset begins in the middle of pt,
1020              do not count first bytes as progress */
1021           GNUNET_assert (app > (dc->offset - sm->offset));
1022           app -= (dc->offset - sm->offset);
1023         }
1024       if (sm->offset + prc->size > dc->offset + dc->length)
1025         {
1026           /* end of block is after relevant range,
1027              do not count last bytes as progress */
1028           GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length));
1029           app -= (sm->offset + prc->size) - (dc->offset + dc->length);
1030         }
1031       dc->completed += app;
1032
1033       /* do recursive download if option is set and either meta data
1034          says it is a directory or if no meta data is given AND filename
1035          ends in '.gnd' (top-level case) */
1036       if (is_recursive_download (dc))
1037         GNUNET_FS_directory_list_contents (prc->size,
1038                                            pt,
1039                                            off,
1040                                            &trigger_recursive_download,
1041                                            dc);
1042
1043     }
1044   pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1045   pi.value.download.specifics.progress.data = pt;
1046   pi.value.download.specifics.progress.offset = sm->offset;
1047   pi.value.download.specifics.progress.data_len = prc->size;
1048   pi.value.download.specifics.progress.depth = sm->depth;
1049   GNUNET_FS_download_make_status_ (&pi, dc);
1050   GNUNET_assert (dc->completed <= dc->length);
1051   if (dc->completed == dc->length)
1052     {
1053 #if DEBUG_DOWNLOAD
1054       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1055                   "Download completed, truncating file to desired length %llu\n",
1056                   (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1057 #endif
1058       /* truncate file to size (since we store IBlocks at the end) */
1059       if (dc->filename != NULL)
1060         {
1061           if (0 != truncate (dc->filename,
1062                              GNUNET_ntohll (dc->uri->data.chk.file_length)))
1063             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1064                                       "truncate",
1065                                       dc->filename);
1066         }
1067       if (dc->job_queue != NULL)
1068         {
1069           GNUNET_FS_dequeue_ (dc->job_queue);
1070           dc->job_queue = NULL;
1071         }
1072       if (is_recursive_download (dc))
1073         full_recursive_download (dc);
1074       if (dc->child_head == NULL)
1075         {
1076           /* signal completion */
1077           pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
1078           GNUNET_FS_download_make_status_ (&pi, dc);
1079           if (dc->parent != NULL)
1080             check_completed (dc->parent);
1081         }
1082       GNUNET_assert (sm->depth == dc->treedepth);
1083     }
1084   if (sm->depth == dc->treedepth)
1085     {
1086       GNUNET_FS_download_sync_ (dc);
1087       GNUNET_free (sm);
1088       return GNUNET_YES;
1089     }
1090 #if DEBUG_DOWNLOAD
1091   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1092               "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1093               sm->depth,
1094               (unsigned long long) sm->offset);
1095 #endif
1096   GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1097   chk = (struct ContentHashKey*) pt;
1098   for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1099     {
1100       off = compute_dblock_offset (sm->offset,
1101                                    sm->depth,
1102                                    dc->treedepth,
1103                                    i);
1104       if ( (off + DBLOCK_SIZE >= dc->offset) &&
1105            (off < dc->offset + dc->length) )
1106         schedule_block_download (dc,
1107                                  &chk[i],
1108                                  off,
1109                                  sm->depth + 1);
1110     }
1111   GNUNET_free (sm);
1112   GNUNET_FS_download_sync_ (dc);
1113   return GNUNET_YES;
1114
1115  signal_error:
1116   if (fh != NULL)
1117     GNUNET_DISK_file_close (fh);
1118   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1119   pi.value.download.specifics.error.message = dc->emsg;
1120   GNUNET_FS_download_make_status_ (&pi, dc);
1121   /* abort all pending requests */
1122   if (NULL != dc->th)
1123     {
1124       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1125       dc->th = NULL;
1126     }
1127   GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1128   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1129                                          &free_entry,
1130                                          NULL);
1131   dc->pending = NULL;
1132   dc->client = NULL;
1133   GNUNET_free (sm);
1134   GNUNET_FS_download_sync_ (dc);
1135   return GNUNET_NO;
1136 }
1137
1138
1139 /**
1140  * Process a download result.
1141  *
1142  * @param dc our download context
1143  * @param type type of the result
1144  * @param data the (encrypted) response
1145  * @param size size of data
1146  */
1147 static void
1148 process_result (struct GNUNET_FS_DownloadContext *dc,
1149                 enum GNUNET_BLOCK_Type type,
1150                 const void *data,
1151                 size_t size)
1152 {
1153   struct ProcessResultClosure prc;
1154
1155   prc.dc = dc;
1156   prc.data = data;
1157   prc.size = size;
1158   prc.type = type;
1159   prc.do_store = GNUNET_YES;
1160   GNUNET_CRYPTO_hash (data, size, &prc.query);
1161 #if DEBUG_DOWNLOAD
1162   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1163               "Received result for query `%s' from `%s'-service\n",
1164               GNUNET_h2s (&prc.query),
1165               "FS");
1166 #endif
1167   GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1168                                               &prc.query,
1169                                               &process_result_with_request,
1170                                               &prc);
1171 }
1172
1173
1174 /**
1175  * Type of a function to call when we receive a message
1176  * from the service.
1177  *
1178  * @param cls closure
1179  * @param msg message received, NULL on timeout or fatal error
1180  */
1181 static void
1182 receive_results (void *cls,
1183                  const struct GNUNET_MessageHeader * msg)
1184 {
1185   struct GNUNET_FS_DownloadContext *dc = cls;
1186   const struct PutMessage *cm;
1187   uint16_t msize;
1188
1189   if ( (NULL == msg) ||
1190        (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1191        (sizeof (struct PutMessage) > ntohs(msg->size)) )
1192     {
1193       GNUNET_break (msg == NULL);
1194       try_reconnect (dc);
1195       return;
1196     }
1197   msize = ntohs(msg->size);
1198   cm = (const struct PutMessage*) msg;
1199   process_result (dc,
1200                   ntohl (cm->type),
1201                   &cm[1],
1202                   msize - sizeof (struct PutMessage));
1203   if (dc->client == NULL)
1204     return; /* fatal error */
1205   /* continue receiving */
1206   GNUNET_CLIENT_receive (dc->client,
1207                          &receive_results,
1208                          dc,
1209                          GNUNET_TIME_UNIT_FOREVER_REL);
1210 }
1211
1212
1213
1214 /**
1215  * We're ready to transmit a search request to the
1216  * file-sharing service.  Do it.  If there is
1217  * more than one request pending, try to send
1218  * multiple or request another transmission.
1219  *
1220  * @param cls closure
1221  * @param size number of bytes available in buf
1222  * @param buf where the callee should write the message
1223  * @return number of bytes written to buf
1224  */
1225 static size_t
1226 transmit_download_request (void *cls,
1227                            size_t size,
1228                            void *buf)
1229 {
1230   struct GNUNET_FS_DownloadContext *dc = cls;
1231   size_t msize;
1232   struct SearchMessage *sm;
1233
1234   dc->th = NULL;
1235   if (NULL == buf)
1236     {
1237 #if DEBUG_DOWNLOAD
1238       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1239                   "Transmitting download request failed, trying to reconnect\n");
1240 #endif
1241       try_reconnect (dc);
1242       return 0;
1243     }
1244   GNUNET_assert (size >= sizeof (struct SearchMessage));
1245   msize = 0;
1246   sm = buf;
1247   while ( (dc->pending != NULL) &&
1248           (size >= msize + sizeof (struct SearchMessage)) )
1249     {
1250 #if DEBUG_DOWNLOAD
1251       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1252                   "Transmitting download request for `%s' to `%s'-service\n",
1253                   GNUNET_h2s (&dc->pending->chk.query),
1254                   "FS");
1255 #endif
1256       memset (sm, 0, sizeof (struct SearchMessage));
1257       sm->header.size = htons (sizeof (struct SearchMessage));
1258       sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1259       if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1260         sm->options = htonl (1);
1261       else
1262         sm->options = htonl (0);
1263       if (dc->pending->depth == dc->treedepth)
1264         sm->type = htonl (GNUNET_BLOCK_TYPE_DBLOCK);
1265       else
1266         sm->type = htonl (GNUNET_BLOCK_TYPE_IBLOCK);
1267       sm->anonymity_level = htonl (dc->anonymity);
1268       sm->target = dc->target.hashPubKey;
1269       sm->query = dc->pending->chk.query;
1270       dc->pending->is_pending = GNUNET_NO;
1271       dc->pending = dc->pending->next;
1272       msize += sizeof (struct SearchMessage);
1273       sm++;
1274     }
1275   if (dc->pending != NULL)
1276     dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1277                                                   sizeof (struct SearchMessage),
1278                                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1279                                                   GNUNET_NO,
1280                                                   &transmit_download_request,
1281                                                   dc);
1282   return msize;
1283 }
1284
1285
1286 /**
1287  * Reconnect to the FS service and transmit our queries NOW.
1288  *
1289  * @param cls our download context
1290  * @param tc unused
1291  */
1292 static void
1293 do_reconnect (void *cls,
1294               const struct GNUNET_SCHEDULER_TaskContext *tc)
1295 {
1296   struct GNUNET_FS_DownloadContext *dc = cls;
1297   struct GNUNET_CLIENT_Connection *client;
1298
1299   dc->task = GNUNET_SCHEDULER_NO_TASK;
1300   client = GNUNET_CLIENT_connect (dc->h->sched,
1301                                   "fs",
1302                                   dc->h->cfg);
1303   if (NULL == client)
1304     {
1305       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1306                   "Connecting to `%s'-service failed, will try again.\n",
1307                   "FS");
1308       try_reconnect (dc);
1309       return;
1310     }
1311   dc->client = client;
1312   dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1313                                                 sizeof (struct SearchMessage),
1314                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1315                                                 GNUNET_NO,
1316                                                 &transmit_download_request,
1317                                                 dc);
1318   GNUNET_CLIENT_receive (client,
1319                          &receive_results,
1320                          dc,
1321                          GNUNET_TIME_UNIT_FOREVER_REL);
1322 }
1323
1324
1325 /**
1326  * Add entries that are not yet pending back to the pending list.
1327  *
1328  * @param cls our download context
1329  * @param key unused
1330  * @param entry entry of type "struct DownloadRequest"
1331  * @return GNUNET_OK
1332  */
1333 static int
1334 retry_entry (void *cls,
1335              const GNUNET_HashCode *key,
1336              void *entry)
1337 {
1338   struct GNUNET_FS_DownloadContext *dc = cls;
1339   struct DownloadRequest *dr = entry;
1340
1341   if (! dr->is_pending)
1342     {
1343       dr->next = dc->pending;
1344       dr->is_pending = GNUNET_YES;
1345       dc->pending = entry;
1346     }
1347   return GNUNET_OK;
1348 }
1349
1350
1351 /**
1352  * We've lost our connection with the FS service.
1353  * Re-establish it and re-transmit all of our
1354  * pending requests.
1355  *
1356  * @param dc download context that is having trouble
1357  */
1358 static void
1359 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1360 {
1361
1362   if (NULL != dc->client)
1363     {
1364 #if DEBUG_DOWNLOAD
1365       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1366                   "Moving all requests back to pending list\n");
1367 #endif
1368       if (NULL != dc->th)
1369         {
1370           GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1371           dc->th = NULL;
1372         }
1373       GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1374                                              &retry_entry,
1375                                              dc);
1376       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1377       dc->client = NULL;
1378     }
1379 #if DEBUG_DOWNLOAD
1380   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1381               "Will try to reconnect in 1s\n");
1382 #endif
1383   dc->task
1384     = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
1385                                     GNUNET_TIME_UNIT_SECONDS,
1386                                     &do_reconnect,
1387                                     dc);
1388 }
1389
1390
1391
1392 /**
1393  * We're allowed to ask the FS service for our blocks.  Start the download.
1394  *
1395  * @param cls the 'struct GNUNET_FS_DownloadContext'
1396  * @param client handle to use for communcation with FS (we must destroy it!)
1397  */
1398 static void
1399 activate_fs_download (void *cls,
1400                       struct GNUNET_CLIENT_Connection *client)
1401 {
1402   struct GNUNET_FS_DownloadContext *dc = cls;
1403   struct GNUNET_FS_ProgressInfo pi;
1404
1405 #if DEBUG_DOWNLOAD
1406   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1407               "Download activated\n");
1408 #endif
1409   GNUNET_assert (NULL != client);
1410   GNUNET_assert (dc->client == NULL);
1411   GNUNET_assert (dc->th == NULL);
1412   dc->client = client;
1413   GNUNET_CLIENT_receive (client,
1414                          &receive_results,
1415                          dc,
1416                          GNUNET_TIME_UNIT_FOREVER_REL);
1417   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1418   GNUNET_FS_download_make_status_ (&pi, dc);
1419   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1420                                          &retry_entry,
1421                                          dc);
1422 #if DEBUG_DOWNLOAD
1423   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1424               "Asking for transmission to FS service\n");
1425 #endif
1426   dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1427                                                 sizeof (struct SearchMessage),
1428                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1429                                                 GNUNET_NO,
1430                                                 &transmit_download_request,
1431                                                 dc);
1432   GNUNET_assert (dc->th != NULL);
1433 }
1434
1435
1436 /**
1437  * We must stop to ask the FS service for our blocks.  Pause the download.
1438  *
1439  * @param cls the 'struct GNUNET_FS_DownloadContext'
1440  */
1441 static void
1442 deactivate_fs_download (void *cls)
1443 {
1444   struct GNUNET_FS_DownloadContext *dc = cls;
1445   struct GNUNET_FS_ProgressInfo pi;
1446
1447 #if DEBUG_DOWNLOAD
1448   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1449               "Download deactivated\n");
1450 #endif
1451   if (NULL != dc->th)
1452     {
1453       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1454       dc->th = NULL;
1455     }
1456   if (NULL != dc->client)
1457     {
1458       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1459       dc->client = NULL;
1460     }
1461   pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1462   GNUNET_FS_download_make_status_ (&pi, dc);
1463 }
1464
1465
1466 /**
1467  * Create SUSPEND event for the given download operation
1468  * and then clean up our state (without stop signal).
1469  *
1470  * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
1471  */
1472 void
1473 GNUNET_FS_download_signal_suspend_ (void *cls)
1474 {
1475   struct GNUNET_FS_DownloadContext *dc = cls;
1476   struct GNUNET_FS_ProgressInfo pi;
1477
1478   if (dc->top != NULL)
1479     GNUNET_FS_end_top (dc->h, dc->top);
1480   while (NULL != dc->child_head)
1481     GNUNET_FS_download_signal_suspend_ (dc->child_head);
1482   if (dc->search != NULL)
1483     {
1484       dc->search->download = NULL;
1485       dc->search = NULL;
1486     }
1487   if (dc->job_queue != NULL)
1488     {
1489       GNUNET_FS_dequeue_ (dc->job_queue);
1490       dc->job_queue = NULL;
1491     }
1492   if (dc->parent != NULL)
1493     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1494                                  dc->parent->child_tail,
1495                                  dc);
1496   pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
1497   GNUNET_FS_download_make_status_ (&pi, dc);
1498   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1499     GNUNET_SCHEDULER_cancel (dc->h->sched,
1500                              dc->task);
1501   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1502                                          &free_entry,
1503                                          NULL);
1504   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1505   GNUNET_free_non_null (dc->filename);
1506   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1507   GNUNET_FS_uri_destroy (dc->uri);
1508   GNUNET_free_non_null (dc->temp_filename);
1509   GNUNET_free_non_null (dc->serialization);
1510   GNUNET_free (dc);
1511 }
1512
1513
1514 /**
1515  * Download parts of a file.  Note that this will store
1516  * the blocks at the respective offset in the given file.  Also, the
1517  * download is still using the blocking of the underlying FS
1518  * encoding.  As a result, the download may *write* outside of the
1519  * given boundaries (if offset and length do not match the 32k FS
1520  * block boundaries). <p>
1521  *
1522  * This function should be used to focus a download towards a
1523  * particular portion of the file (optimization), not to strictly
1524  * limit the download to exactly those bytes.
1525  *
1526  * @param h handle to the file sharing subsystem
1527  * @param uri the URI of the file (determines what to download); CHK or LOC URI
1528  * @param meta known metadata for the file (can be NULL)
1529  * @param filename where to store the file, maybe NULL (then no file is
1530  *        created on disk and data must be grabbed from the callbacks)
1531  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1532  *        can be NULL (in which case we will pick a name if needed); the temporary file
1533  *        may already exist, in which case we will try to use the data that is there and
1534  *        if it is not what is desired, will overwrite it
1535  * @param offset at what offset should we start the download (typically 0)
1536  * @param length how many bytes should be downloaded starting at offset
1537  * @param anonymity anonymity level to use for the download
1538  * @param options various options
1539  * @param cctx initial value for the client context for this download
1540  * @param parent parent download to associate this download with (use NULL
1541  *        for top-level downloads; useful for manually-triggered recursive downloads)
1542  * @return context that can be used to control this download
1543  */
1544 struct GNUNET_FS_DownloadContext *
1545 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
1546                           const struct GNUNET_FS_Uri *uri,
1547                           const struct GNUNET_CONTAINER_MetaData *meta,
1548                           const char *filename,
1549                           const char *tempname,
1550                           uint64_t offset,
1551                           uint64_t length,
1552                           uint32_t anonymity,
1553                           enum GNUNET_FS_DownloadOptions options,
1554                           void *cctx,
1555                           struct GNUNET_FS_DownloadContext *parent)
1556 {
1557   struct GNUNET_FS_ProgressInfo pi;
1558   struct GNUNET_FS_DownloadContext *dc;
1559
1560   GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
1561   if ( (offset + length < offset) ||
1562        (offset + length > uri->data.chk.file_length) )
1563     {
1564       GNUNET_break (0);
1565       return NULL;
1566     }
1567 #if DEBUG_DOWNLOAD
1568   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1569               "Starting download `%s' of %llu bytes\n",
1570               filename,
1571               (unsigned long long) length);
1572 #endif
1573   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1574   dc->h = h;
1575   dc->parent = parent;
1576   if (parent != NULL)
1577     {
1578       GNUNET_CONTAINER_DLL_insert (parent->child_head,
1579                                    parent->child_tail,
1580                                    dc);
1581     }
1582   dc->uri = GNUNET_FS_uri_dup (uri);
1583   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1584   dc->client_info = cctx;
1585   dc->start_time = GNUNET_TIME_absolute_get ();
1586   if (NULL != filename)
1587     {
1588       dc->filename = GNUNET_strdup (filename);
1589       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1590         GNUNET_DISK_file_size (filename,
1591                                &dc->old_file_size,
1592                                GNUNET_YES);
1593     }
1594   if (GNUNET_FS_uri_test_loc (dc->uri))
1595     GNUNET_assert (GNUNET_OK ==
1596                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1597                                                         &dc->target));
1598   dc->offset = offset;
1599   dc->length = length;
1600   dc->anonymity = anonymity;
1601   dc->options = options;
1602   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1603   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1604   if ( (filename == NULL) &&
1605        (is_recursive_download (dc) ) )
1606     {
1607       if (tempname != NULL)
1608         dc->temp_filename = GNUNET_strdup (tempname);
1609       else
1610         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1611     }
1612
1613 #if DEBUG_DOWNLOAD
1614   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1615               "Download tree has depth %u\n",
1616               dc->treedepth);
1617 #endif
1618   if (parent == NULL)
1619     {
1620       dc->top = GNUNET_FS_make_top (dc->h,
1621                                     &GNUNET_FS_download_signal_suspend_,
1622                                     dc);
1623     }
1624   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1625   pi.value.download.specifics.start.meta = meta;
1626   GNUNET_FS_download_make_status_ (&pi, dc);
1627   schedule_block_download (dc,
1628                            &dc->uri->data.chk.chk,
1629                            0,
1630                            1 /* 0 == CHK, 1 == top */);
1631   GNUNET_FS_download_sync_ (dc);
1632   GNUNET_FS_download_start_downloading_ (dc);
1633   return dc;
1634 }
1635
1636
1637 /**
1638  * Download parts of a file based on a search result.  The download
1639  * will be associated with the search result (and the association
1640  * will be preserved when serializing/deserializing the state).
1641  * If the search is stopped, the download will not be aborted but
1642  * be 'promoted' to a stand-alone download.
1643  *
1644  * As with the other download function, this will store
1645  * the blocks at the respective offset in the given file.  Also, the
1646  * download is still using the blocking of the underlying FS
1647  * encoding.  As a result, the download may *write* outside of the
1648  * given boundaries (if offset and length do not match the 32k FS
1649  * block boundaries). <p>
1650  *
1651  * The given range can be used to focus a download towards a
1652  * particular portion of the file (optimization), not to strictly
1653  * limit the download to exactly those bytes.
1654  *
1655  * @param h handle to the file sharing subsystem
1656  * @param sr the search result to use for the download (determines uri and
1657  *        meta data and associations)
1658  * @param filename where to store the file, maybe NULL (then no file is
1659  *        created on disk and data must be grabbed from the callbacks)
1660  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1661  *        can be NULL (in which case we will pick a name if needed); the temporary file
1662  *        may already exist, in which case we will try to use the data that is there and
1663  *        if it is not what is desired, will overwrite it
1664  * @param offset at what offset should we start the download (typically 0)
1665  * @param length how many bytes should be downloaded starting at offset
1666  * @param anonymity anonymity level to use for the download
1667  * @param options various download options
1668  * @param cctx initial value for the client context for this download
1669  * @return context that can be used to control this download
1670  */
1671 struct GNUNET_FS_DownloadContext *
1672 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
1673                                       struct GNUNET_FS_SearchResult *sr,
1674                                       const char *filename,
1675                                       const char *tempname,
1676                                       uint64_t offset,
1677                                       uint64_t length,
1678                                       uint32_t anonymity,
1679                                       enum GNUNET_FS_DownloadOptions options,
1680                                       void *cctx)
1681 {
1682   struct GNUNET_FS_ProgressInfo pi;
1683   struct GNUNET_FS_DownloadContext *dc;
1684
1685   if ( (sr == NULL) ||
1686        (sr->download != NULL) )
1687     {
1688       GNUNET_break (0);
1689       return NULL;
1690     }
1691   GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri));
1692   if ( (offset + length < offset) ||
1693        (offset + length > sr->uri->data.chk.file_length) )
1694     {
1695       GNUNET_break (0);
1696       return NULL;
1697     }
1698 #if DEBUG_DOWNLOAD
1699   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1700               "Starting download `%s' of %llu bytes\n",
1701               filename,
1702               (unsigned long long) length);
1703 #endif
1704   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1705   dc->h = h;
1706   dc->search = sr;
1707   sr->download = dc;
1708   if (sr->probe_ctx != NULL)
1709     {
1710       GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
1711       sr->probe_ctx = NULL;
1712     }
1713   dc->uri = GNUNET_FS_uri_dup (sr->uri);
1714   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
1715   dc->client_info = cctx;
1716   dc->start_time = GNUNET_TIME_absolute_get ();
1717   if (NULL != filename)
1718     {
1719       dc->filename = GNUNET_strdup (filename);
1720       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1721         GNUNET_DISK_file_size (filename,
1722                                &dc->old_file_size,
1723                                GNUNET_YES);
1724     }
1725   if (GNUNET_FS_uri_test_loc (dc->uri))
1726     GNUNET_assert (GNUNET_OK ==
1727                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1728                                                         &dc->target));
1729   dc->offset = offset;
1730   dc->length = length;
1731   dc->anonymity = anonymity;
1732   dc->options = options;
1733   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1734   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1735   if ( (filename == NULL) &&
1736        (is_recursive_download (dc) ) )
1737     {
1738       if (tempname != NULL)
1739         dc->temp_filename = GNUNET_strdup (tempname);
1740       else
1741         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1742     }
1743
1744 #if DEBUG_DOWNLOAD
1745   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1746               "Download tree has depth %u\n",
1747               dc->treedepth);
1748 #endif
1749   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1750   pi.value.download.specifics.start.meta = dc->meta;
1751   GNUNET_FS_download_make_status_ (&pi, dc);
1752   schedule_block_download (dc,
1753                            &dc->uri->data.chk.chk,
1754                            0,
1755                            1 /* 0 == CHK, 1 == top */);
1756   GNUNET_FS_download_sync_ (dc);
1757   GNUNET_FS_download_start_downloading_ (dc);
1758   return dc;
1759 }
1760
1761
1762 /**
1763  * Start the downloading process (by entering the queue).
1764  *
1765  * @param dc our download context
1766  */
1767 void
1768 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
1769 {
1770   GNUNET_assert (dc->job_queue == NULL);
1771   dc->job_queue = GNUNET_FS_queue_ (dc->h,
1772                                     &activate_fs_download,
1773                                     &deactivate_fs_download,
1774                                     dc,
1775                                     (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
1776 }
1777
1778
1779 /**
1780  * Stop a download (aborts if download is incomplete).
1781  *
1782  * @param dc handle for the download
1783  * @param do_delete delete files of incomplete downloads
1784  */
1785 void
1786 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
1787                          int do_delete)
1788 {
1789   struct GNUNET_FS_ProgressInfo pi;
1790   int have_children;
1791
1792   if (dc->top != NULL)
1793     GNUNET_FS_end_top (dc->h, dc->top);
1794   if (dc->search != NULL)
1795     {
1796       dc->search->download = NULL;
1797       dc->search = NULL;
1798     }
1799   if (dc->job_queue != NULL)
1800     {
1801       GNUNET_FS_dequeue_ (dc->job_queue);
1802       dc->job_queue = NULL;
1803     }
1804   have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
1805   while (NULL != dc->child_head)
1806     GNUNET_FS_download_stop (dc->child_head,
1807                              do_delete);
1808   if (dc->parent != NULL)
1809     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1810                                  dc->parent->child_tail,
1811                                  dc);
1812   if (dc->serialization != NULL)
1813     GNUNET_FS_remove_sync_file_ (dc->h,
1814                                  ( (dc->parent != NULL)  || (dc->search != NULL) )
1815                                  ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1816                                  : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
1817                                  dc->serialization);
1818   if ( (GNUNET_YES == have_children) &&
1819        (dc->parent == NULL) )
1820     GNUNET_FS_remove_sync_dir_ (dc->h,
1821                                 (dc->search != NULL)
1822                                 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1823                                 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
1824                                 dc->serialization);
1825   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
1826   GNUNET_FS_download_make_status_ (&pi, dc);
1827   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1828     GNUNET_SCHEDULER_cancel (dc->h->sched,
1829                              dc->task);
1830   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1831                                          &free_entry,
1832                                          NULL);
1833   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1834   if (dc->filename != NULL)
1835     {
1836       if ( (dc->completed != dc->length) &&
1837            (GNUNET_YES == do_delete) )
1838         {
1839           if (0 != UNLINK (dc->filename))
1840             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1841                                       "unlink",
1842                                       dc->filename);
1843         }
1844       GNUNET_free (dc->filename);
1845     }
1846   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1847   GNUNET_FS_uri_destroy (dc->uri);
1848   if (NULL != dc->temp_filename)
1849     {
1850       if (0 != UNLINK (dc->temp_filename))
1851         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1852                                   "unlink",
1853                                   dc->temp_filename);
1854       GNUNET_free (dc->temp_filename);
1855     }
1856   GNUNET_free_non_null (dc->serialization);
1857   GNUNET_free (dc);
1858 }
1859
1860 /* end of fs_download.c */