src/fs/fs_download.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file fs/fs_download.c
  22  * @brief download methods
  23  * @author Christian Grothoff
  24  *
  25  * TODO:
  26  * - different priority for scheduling probe downloads?
  27  * - check if iblocks can be computed from existing blocks (can wait, hard)
  28  */
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_fs_service.h"
  32 #include "fs.h"
  33 #include "fs_tree.h"
  34
  35 #define DEBUG_DOWNLOAD GNUNET_NO
  36
  37 /**
  38  * Determine if the given download (options and meta data) should cause
  39  * use to try to do a recursive download.
  40  */
  41 static int
  42 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
  43 {
  44   return  (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
  45     ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
  46       ( (dc->meta == NULL) &&
  47         ( (NULL == dc->filename) ||
  48           ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
  49             (NULL !=
  50              strstr (dc->filename + strlen(dc->filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
  51                      GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
  52 }
  53
  54
  55 /**
  56  * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
  57  * only have to truncate the file once we're done).
  58  *
  59  * Given the offset of a block (with respect to the DBLOCKS) and its
  60  * depth, return the offset where we would store this block in the
  61  * file.
  62  *
  63  * @param fsize overall file size
  64  * @param off offset of the block in the file
  65  * @param depth depth of the block in the tree
  66  * @param treedepth maximum depth of the tree
  67  * @return off for DBLOCKS (depth == treedepth),
  68  *         otherwise an offset past the end
  69  *         of the file that does not overlap
  70  *         with the range for any other block
  71  */
  72 static uint64_t
  73 compute_disk_offset (uint64_t fsize,
  74                      uint64_t off,
  75                      unsigned int depth,
  76                      unsigned int treedepth)
  77 {
  78   unsigned int i;
  79   uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
  80   uint64_t loff; /* where do IBlocks for depth "i" start? */
  81   unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
  82
  83   if (depth == treedepth)
  84     return off;
  85   /* first IBlocks start at the end of file, rounded up
  86      to full DBLOCK_SIZE */
  87   loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
  88   lsize = ( (fsize + DBLOCK_SIZE-1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey);
  89   GNUNET_assert (0 == (off % DBLOCK_SIZE));
  90   ioff = (off / DBLOCK_SIZE);
  91   for (i=treedepth-1;i>depth;i--)
  92     {
  93       loff += lsize;
  94       lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
  95       GNUNET_assert (lsize > 0);
  96       GNUNET_assert (0 == (ioff % CHK_PER_INODE));
  97       ioff /= CHK_PER_INODE;
  98     }
  99   return loff + ioff * sizeof (struct ContentHashKey);
 100 }
 101
 102
 103 /**
 104  * Given a file of the specified treedepth and a block at the given
 105  * offset and depth, calculate the offset for the CHK at the given
 106  * index.
 107  *
 108  * @param offset the offset of the first
 109  *        DBLOCK in the subtree of the
 110  *        identified IBLOCK
 111  * @param depth the depth of the IBLOCK in the tree
 112  * @param treedepth overall depth of the tree
 113  * @param k which CHK in the IBLOCK are we
 114  *        talking about
 115  * @return offset if k=0, otherwise an appropriately
 116  *         larger value (i.e., if depth = treedepth-1,
 117  *         the returned value should be offset+DBLOCK_SIZE)
 118  */
 119 static uint64_t
 120 compute_dblock_offset (uint64_t offset,
 121                        unsigned int depth,
 122                        unsigned int treedepth,
 123                        unsigned int k)
 124 {
 125   unsigned int i;
 126   uint64_t lsize; /* what is the size of the sum of all DBlocks
 127                      that a CHK at depth i corresponds to? */
 128
 129   if (depth == treedepth)
 130     return offset;
 131   lsize = DBLOCK_SIZE;
 132   for (i=treedepth-1;i>depth;i--)
 133     lsize *= CHK_PER_INODE;
 134   return offset + k * lsize;
 135 }
 136
 137
 138 /**
 139  * Fill in all of the generic fields for a download event and call the
 140  * callback.
 141  *
 142  * @param pi structure to fill in
 143  * @param dc overall download context
 144  */
 145 void
 146 GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
 147                                  struct GNUNET_FS_DownloadContext *dc)
 148 {
 149   pi->value.download.dc = dc;
 150   pi->value.download.cctx
 151     = dc->client_info;
 152   pi->value.download.pctx
 153     = (dc->parent == NULL) ? NULL : dc->parent->client_info;
 154   pi->value.download.sctx
 155     = (dc->search == NULL) ? NULL : dc->search->client_info;
 156   pi->value.download.uri
 157     = dc->uri;
 158   pi->value.download.filename
 159     = dc->filename;
 160   pi->value.download.size
 161     = dc->length;
 162   pi->value.download.duration
 163     = GNUNET_TIME_absolute_get_duration (dc->start_time);
 164   pi->value.download.completed
 165     = dc->completed;
 166   pi->value.download.anonymity
 167     = dc->anonymity;
 168   pi->value.download.eta
 169     = GNUNET_TIME_calculate_eta (dc->start_time,
 170                                  dc->completed,
 171                                  dc->length);
 172   pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES;
 173   if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
 174     dc->client_info = dc->h->upcb (dc->h->upcb_cls,
 175                                    pi);
 176   else
 177     dc->client_info = GNUNET_FS_search_probe_progress_ (NULL,
 178                                                         pi);
 179 }
 180
 181 /**
 182  * We're ready to transmit a search request to the
 183  * file-sharing service.  Do it.  If there is
 184  * more than one request pending, try to send
 185  * multiple or request another transmission.
 186  *
 187  * @param cls closure
 188  * @param size number of bytes available in buf
 189  * @param buf where the callee should write the message
 190  * @return number of bytes written to buf
 191  */
 192 static size_t
 193 transmit_download_request (void *cls,
 194                            size_t size,
 195                            void *buf);
 196
 197
 198 /**
 199  * Closure for iterator processing results.
 200  */
 201 struct ProcessResultClosure
 202 {
 203
 204   /**
 205    * Hash of data.
 206    */
 207   GNUNET_HashCode query;
 208
 209   /**
 210    * Data found in P2P network.
 211    */
 212   const void *data;
 213
 214   /**
 215    * Our download context.
 216    */
 217   struct GNUNET_FS_DownloadContext *dc;
 218
 219   /**
 220    * Number of bytes in data.
 221    */
 222   size_t size;
 223
 224   /**
 225    * Type of data.
 226    */
 227   enum GNUNET_BLOCK_Type type;
 228
 229   /**
 230    * Flag to indicate if this block should be stored on disk.
 231    */
 232   int do_store;
 233
 234 };
 235
 236
 237 /**
 238  * Iterator over entries in the pending requests in the 'active' map for the
 239  * reply that we just got.
 240  *
 241  * @param cls closure (our 'struct ProcessResultClosure')
 242  * @param key query for the given value / request
 243  * @param value value in the hash map (a 'struct DownloadRequest')
 244  * @return GNUNET_YES (we should continue to iterate); unless serious error
 245  */
 246 static int
 247 process_result_with_request (void *cls,
 248                              const GNUNET_HashCode * key,
 249                              void *value);
 250
 251
 252
 253 /**
 254  * Schedule the download of the specified block in the tree.
 255  *
 256  * @param dc overall download this block belongs to
 257  * @param chk content-hash-key of the block
 258  * @param offset offset of the block in the file
 259  *         (for IBlocks, the offset is the lowest
 260  *          offset of any DBlock in the subtree under
 261  *          the IBlock)
 262  * @param depth depth of the block, 0 is the root of the tree
 263  */
 264 static void
 265 schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
 266                          const struct ContentHashKey *chk,
 267                          uint64_t offset,
 268                          unsigned int depth)
 269 {
 270   struct DownloadRequest *sm;
 271   uint64_t total;
 272   uint64_t off;
 273   size_t len;
 274   char block[DBLOCK_SIZE];
 275   GNUNET_HashCode key;
 276   struct ProcessResultClosure prc;
 277   struct GNUNET_DISK_FileHandle *fh;
 278
 279 #if DEBUG_DOWNLOAD
 280   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 281               "Scheduling download at offset %llu and depth %u for `%s'\n",
 282               (unsigned long long) offset,
 283               depth,
 284               GNUNET_h2s (&chk->query));
 285 #endif
 286   total = GNUNET_ntohll (dc->uri->data.chk.file_length);
 287   off = compute_disk_offset (total,
 288                              offset,
 289                              depth,
 290                              dc->treedepth);
 291   len = GNUNET_FS_tree_calculate_block_size (total,
 292                                              dc->treedepth,
 293                                              offset,
 294                                              depth);
 295   sm = GNUNET_malloc (sizeof (struct DownloadRequest));
 296   sm->chk = *chk;
 297   sm->offset = offset;
 298   sm->depth = depth;
 299   sm->is_pending = GNUNET_YES;
 300   sm->next = dc->pending;
 301   dc->pending = sm;
 302   GNUNET_CONTAINER_multihashmap_put (dc->active,
 303                                      &chk->query,
 304                                      sm,
 305                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
 306   fh = NULL;
 307   if ( (dc->old_file_size > off) &&
 308        (dc->filename != NULL) )
 309     fh = GNUNET_DISK_file_open (dc->filename,
 310                                 GNUNET_DISK_OPEN_READ,
 311                                 GNUNET_DISK_PERM_NONE);
 312   if ( (fh != NULL) &&
 313        (off  ==
 314         GNUNET_DISK_file_seek (fh,
 315                                off,
 316                                GNUNET_DISK_SEEK_SET) ) &&
 317        (len ==
 318         GNUNET_DISK_file_read (fh,
 319                                block,
 320                                len)) )
 321     {
 322       GNUNET_CRYPTO_hash (block, len, &key);
 323       if (0 == memcmp (&key,
 324                        &chk->key,
 325                        sizeof (GNUNET_HashCode)))
 326         {
 327           char enc[len];
 328           struct GNUNET_CRYPTO_AesSessionKey sk;
 329           struct GNUNET_CRYPTO_AesInitializationVector iv;
 330           GNUNET_HashCode query;
 331
 332           GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
 333           if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len,
 334                                                &sk,
 335                                                &iv,
 336                                                enc))
 337             {
 338               GNUNET_break (0);
 339               goto do_download;
 340             }
 341           GNUNET_CRYPTO_hash (enc, len, &query);
 342           if (0 == memcmp (&query,
 343                            &chk->query,
 344                            sizeof (GNUNET_HashCode)))
 345             {
 346 #if DEBUG_DOWNLOAD
 347               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 348                           "Matching block already present, no need for download!\n");
 349 #endif
 350               /* already got it! */
 351               prc.dc = dc;
 352               prc.data = enc;
 353               prc.size = len;
 354               prc.type = (dc->treedepth == depth)
 355                 ? GNUNET_BLOCK_TYPE_DBLOCK
 356                 : GNUNET_BLOCK_TYPE_IBLOCK;
 357               prc.query = chk->query;
 358               prc.do_store = GNUNET_NO; /* useless */
 359               process_result_with_request (&prc,
 360                                            &key,
 361                                            sm);
 362             }
 363           else
 364             {
 365               GNUNET_break_op (0);
 366             }
 367           GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 368           return;
 369         }
 370     }
 371  do_download:
 372   if (fh != NULL)
 373     GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
 374   if (depth < dc->treedepth)
 375     {
 376       // FIXME: try if we could
 377       // reconstitute this IBLOCK
 378       // from the existing blocks on disk (can wait)
 379       // (read block(s), encode, compare with
 380       // query; if matches, simply return)
 381     }
 382
 383   if ( (dc->th == NULL) &&
 384        (dc->client != NULL) )
 385     {
 386 #if DEBUG_DOWNLOAD
 387       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 388                   "Asking for transmission to FS service\n");
 389 #endif
 390       dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
 391                                                     sizeof (struct SearchMessage),
 392                                                     GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 393                                                     GNUNET_NO,
 394                                                     &transmit_download_request,
 395                                                     dc);
 396     }
 397   else
 398     {
 399 #if DEBUG_DOWNLOAD
 400       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 401                   "Transmission request not issued (%p %p)\n",
 402                   dc->th,
 403                   dc->client);
 404 #endif
 405
 406     }
 407
 408 }
 409
 410
 411
 412 /**
 413  * Suggest a filename based on given metadata.
 414  *
 415  * @param md given meta data
 416  * @return NULL if meta data is useless for suggesting a filename
 417  */
 418 char *
 419 GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData *md)
 420 {
 421   static const char *mimeMap[][2] = {
 422     {"application/bz2", ".bz2"},
 423     {"application/gnunet-directory", ".gnd"},
 424     {"application/java", ".class"},
 425     {"application/msword", ".doc"},
 426     {"application/ogg", ".ogg"},
 427     {"application/pdf", ".pdf"},
 428     {"application/pgp-keys", ".key"},
 429     {"application/pgp-signature", ".pgp"},
 430     {"application/postscript", ".ps"},
 431     {"application/rar", ".rar"},
 432     {"application/rtf", ".rtf"},
 433     {"application/xml", ".xml"},
 434     {"application/x-debian-package", ".deb"},
 435     {"application/x-dvi", ".dvi"},
 436     {"applixation/x-flac", ".flac"},
 437     {"applixation/x-gzip", ".gz"},
 438     {"application/x-java-archive", ".jar"},
 439     {"application/x-java-vm", ".class"},
 440     {"application/x-python-code", ".pyc"},
 441     {"application/x-redhat-package-manager", ".rpm"},
 442     {"application/x-rpm", ".rpm"},
 443     {"application/x-tar", ".tar"},
 444     {"application/x-tex-pk", ".pk"},
 445     {"application/x-texinfo", ".texinfo"},
 446     {"application/x-xcf", ".xcf"},
 447     {"application/x-xfig", ".xfig"},
 448     {"application/zip", ".zip"},
 449
 450     {"audio/midi", ".midi"},
 451     {"audio/mpeg", ".mp3"},
 452     {"audio/real", ".rm"},
 453     {"audio/x-wav", ".wav"},
 454
 455     {"image/gif", ".gif"},
 456     {"image/jpeg", ".jpg"},
 457     {"image/pcx", ".pcx"},
 458     {"image/png", ".png"},
 459     {"image/tiff", ".tiff"},
 460     {"image/x-ms-bmp", ".bmp"},
 461     {"image/x-xpixmap", ".xpm"},
 462
 463     {"text/css", ".css"},
 464     {"text/html", ".html"},
 465     {"text/plain", ".txt"},
 466     {"text/rtf", ".rtf"},
 467     {"text/x-c++hdr", ".h++"},
 468     {"text/x-c++src", ".c++"},
 469     {"text/x-chdr", ".h"},
 470     {"text/x-csrc", ".c"},
 471     {"text/x-java", ".java"},
 472     {"text/x-moc", ".moc"},
 473     {"text/x-pascal", ".pas"},
 474     {"text/x-perl", ".pl"},
 475     {"text/x-python", ".py"},
 476     {"text/x-tex", ".tex"},
 477
 478     {"video/avi", ".avi"},
 479     {"video/mpeg", ".mpeg"},
 480     {"video/quicktime", ".qt"},
 481     {"video/real", ".rm"},
 482     {"video/x-msvideo", ".avi"},
 483     {NULL, NULL},
 484   };
 485   char *ret;
 486   unsigned int i;
 487   char *mime;
 488   char *base;
 489   const char *ext;
 490
 491   ret = GNUNET_CONTAINER_meta_data_get_by_type (md,
 492                                                 EXTRACTOR_METATYPE_FILENAME);
 493   if (ret != NULL)
 494     return ret;
 495   ext = NULL;
 496   mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
 497                                                  EXTRACTOR_METATYPE_MIMETYPE);
 498   if (mime != NULL)
 499     {
 500       i = 0;
 501       while ( (mimeMap[i][0] != NULL) &&
 502               (0 != strcmp (mime, mimeMap[i][0])))
 503         i++;
 504       if (mimeMap[i][1] == NULL)
 505         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG |
 506                     GNUNET_ERROR_TYPE_BULK,
 507                     _("Did not find mime type `%s' in extension list.\n"),
 508                     mime);
 509       else
 510         ext = mimeMap[i][1];
 511       GNUNET_free (mime);
 512     }
 513   base = GNUNET_CONTAINER_meta_data_get_first_by_types (md,
 514                                                         EXTRACTOR_METATYPE_TITLE,
 515                                                         EXTRACTOR_METATYPE_BOOK_TITLE,
 516                                                         EXTRACTOR_METATYPE_ORIGINAL_TITLE,
 517                                                         EXTRACTOR_METATYPE_PACKAGE_NAME,
 518                                                         EXTRACTOR_METATYPE_URL,
 519                                                         EXTRACTOR_METATYPE_URI,
 520                                                         EXTRACTOR_METATYPE_DESCRIPTION,
 521                                                         EXTRACTOR_METATYPE_ISRC,
 522                                                         EXTRACTOR_METATYPE_JOURNAL_NAME,
 523                                                         EXTRACTOR_METATYPE_AUTHOR_NAME,
 524                                                         EXTRACTOR_METATYPE_SUBJECT,
 525                                                         EXTRACTOR_METATYPE_ALBUM,
 526                                                         EXTRACTOR_METATYPE_ARTIST,
 527                                                         EXTRACTOR_METATYPE_KEYWORDS,
 528                                                         EXTRACTOR_METATYPE_COMMENT,
 529                                                         EXTRACTOR_METATYPE_UNKNOWN,
 530                                                         -1);
 531   if ( (base == NULL) &&
 532        (ext == NULL) )
 533     return NULL;
 534   if (base == NULL)
 535     return GNUNET_strdup (ext);
 536   if (ext == NULL)
 537     return base;
 538   GNUNET_asprintf (&ret,
 539                    "%s%s",
 540                    base,
 541                    ext);
 542   GNUNET_free (base);
 543   return ret;
 544 }
 545
 546
 547 /**
 548  * We've lost our connection with the FS service.
 549  * Re-establish it and re-transmit all of our
 550  * pending requests.
 551  *
 552  * @param dc download context that is having trouble
 553  */
 554 static void
 555 try_reconnect (struct GNUNET_FS_DownloadContext *dc);
 556
 557
 558 /**
 559  * We found an entry in a directory.  Check if the respective child
 560  * already exists and if not create the respective child download.
 561  *
 562  * @param cls the parent download
 563  * @param filename name of the file in the directory
 564  * @param uri URI of the file (CHK or LOC)
 565  * @param meta meta data of the file
 566  * @param length number of bytes in data
 567  * @param data contents of the file (or NULL if they were not inlined)
 568  */
 569 static void
 570 trigger_recursive_download (void *cls,
 571                             const char *filename,
 572                             const struct GNUNET_FS_Uri *uri,
 573                             const struct GNUNET_CONTAINER_MetaData *meta,
 574                             size_t length,
 575                             const void *data);
 576
 577
 578 /**
 579  * We're done downloading a directory.  Open the file and
 580  * trigger all of the (remaining) child downloads.
 581  *
 582  * @param dc context of download that just completed
 583  */
 584 static void
 585 full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 586 {
 587   size_t size;
 588   uint64_t size64;
 589   void *data;
 590   struct GNUNET_DISK_FileHandle *h;
 591   struct GNUNET_DISK_MapHandle *m;
 592
 593   size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
 594   size = (size_t) size64;
 595   if (size64 != (uint64_t) size)
 596     {
 597       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 598                   _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
 599       return;
 600     }
 601   if (dc->filename != NULL)
 602     {
 603       h = GNUNET_DISK_file_open (dc->filename,
 604                                  GNUNET_DISK_OPEN_READ,
 605                                  GNUNET_DISK_PERM_NONE);
 606     }
 607   else
 608     {
 609       GNUNET_assert (dc->temp_filename != NULL);
 610       h = GNUNET_DISK_file_open (dc->temp_filename,
 611                                  GNUNET_DISK_OPEN_READ,
 612                                  GNUNET_DISK_PERM_NONE);
 613     }
 614   if (h == NULL)
 615     return; /* oops */
 616   data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
 617   if (data == NULL)
 618     {
 619       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 620                   _("Directory too large for system address space\n"));
 621     }
 622   else
 623     {
 624       GNUNET_FS_directory_list_contents (size,
 625                                          data,
 626                                          0,
 627                                          &trigger_recursive_download,
 628                                          dc);
 629       GNUNET_DISK_file_unmap (m);
 630     }
 631   GNUNET_DISK_file_close (h);
 632   if (dc->filename == NULL)
 633     {
 634       if (0 != UNLINK (dc->temp_filename))
 635         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
 636                                   "unlink",
 637                                   dc->temp_filename);
 638       GNUNET_free (dc->temp_filename);
 639       dc->temp_filename = NULL;
 640     }
 641 }
 642
 643
 644 /**
 645  * Check if all child-downloads have completed and
 646  * if so, signal completion (and possibly recurse to
 647  * parent).
 648  */
 649 static void
 650 check_completed (struct GNUNET_FS_DownloadContext *dc)
 651 {
 652   struct GNUNET_FS_ProgressInfo pi;
 653   struct GNUNET_FS_DownloadContext *pos;
 654
 655   pos = dc->child_head;
 656   while (pos != NULL)
 657     {
 658       if ( (pos->emsg == NULL) &&
 659            (pos->completed < pos->length) )
 660         return; /* not done yet */
 661       if ( (pos->child_head != NULL) &&
 662            (pos->has_finished != GNUNET_YES) )
 663         return; /* not transitively done yet */
 664       pos = pos->next;
 665     }
 666   dc->has_finished = GNUNET_YES;
 667   GNUNET_FS_download_sync_ (dc);
 668   /* signal completion */
 669   pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
 670   GNUNET_FS_download_make_status_ (&pi, dc);
 671   if (dc->parent != NULL)
 672     check_completed (dc->parent);
 673 }
 674
 675
 676 /**
 677  * We found an entry in a directory.  Check if the respective child
 678  * already exists and if not create the respective child download.
 679  *
 680  * @param cls the parent download
 681  * @param filename name of the file in the directory
 682  * @param uri URI of the file (CHK or LOC)
 683  * @param meta meta data of the file
 684  * @param length number of bytes in data
 685  * @param data contents of the file (or NULL if they were not inlined)
 686  */
 687 static void
 688 trigger_recursive_download (void *cls,
 689                             const char *filename,
 690                             const struct GNUNET_FS_Uri *uri,
 691                             const struct GNUNET_CONTAINER_MetaData *meta,
 692                             size_t length,
 693                             const void *data)
 694 {
 695   struct GNUNET_FS_DownloadContext *dc = cls;
 696   struct GNUNET_FS_DownloadContext *cpos;
 697   struct GNUNET_DISK_FileHandle *fh;
 698   char *temp_name;
 699   const char *real_name;
 700   char *fn;
 701   char *us;
 702   char *ext;
 703   char *dn;
 704   char *pos;
 705   char *full_name;
 706
 707   if (NULL == uri)
 708     return; /* entry for the directory itself */
 709   cpos = dc->child_head;
 710   while (cpos != NULL)
 711     {
 712       if ( (GNUNET_FS_uri_test_equal (uri,
 713                                       cpos->uri)) ||
 714            ( (filename != NULL) &&
 715              (0 == strcmp (cpos->filename,
 716                            filename)) ) )
 717         break;
 718       cpos = cpos->next;
 719     }
 720   if (cpos != NULL)
 721     return; /* already exists */
 722   fn = NULL;
 723   if (NULL == filename)
 724     {
 725       fn = GNUNET_FS_meta_data_suggest_filename (meta);
 726       if (fn == NULL)
 727         {
 728           us = GNUNET_FS_uri_to_string (uri);
 729           fn = GNUNET_strdup (&us [strlen (GNUNET_FS_URI_PREFIX
 730                                            GNUNET_FS_URI_CHK_INFIX)]);
 731           GNUNET_free (us);
 732         }
 733       else if (fn[0] == '.')
 734         {
 735           ext = fn;
 736           us = GNUNET_FS_uri_to_string (uri);
 737           GNUNET_asprintf (&fn,
 738                            "%s%s",
 739                            &us[strlen (GNUNET_FS_URI_PREFIX
 740                                        GNUNET_FS_URI_CHK_INFIX)], ext);
 741           GNUNET_free (ext);
 742           GNUNET_free (us);
 743         }
 744       /* change '\' to '/' (this should have happened
 745        during insertion, but malicious peers may
 746        not have done this) */
 747       while (NULL != (pos = strstr (fn, "\\")))
 748         *pos = '/';
 749       /* remove '../' everywhere (again, well-behaved
 750          peers don't do this, but don't trust that
 751          we did not get something nasty) */
 752       while (NULL != (pos = strstr (fn, "../")))
 753         {
 754           pos[0] = '_';
 755           pos[1] = '_';
 756           pos[2] = '_';
 757         }
 758       filename = fn;
 759     }
 760   if (dc->filename == NULL)
 761     {
 762       full_name = NULL;
 763     }
 764   else
 765     {
 766       dn = GNUNET_strdup (dc->filename);
 767       GNUNET_break ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 768                      (NULL !=
 769                       strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 770                               GNUNET_FS_DIRECTORY_EXT)) );
 771       if ( (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
 772            (NULL !=
 773             strstr (dn + strlen(dn) - strlen(GNUNET_FS_DIRECTORY_EXT),
 774                     GNUNET_FS_DIRECTORY_EXT)) )
 775         dn[strlen(dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
 776       if ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
 777            ( (strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
 778              (NULL ==
 779               strstr (filename + strlen(filename) - strlen(GNUNET_FS_DIRECTORY_EXT),
 780                       GNUNET_FS_DIRECTORY_EXT)) ) )
 781         {
 782           GNUNET_asprintf (&full_name,
 783                            "%s%s%s%s",
 784                            dn,
 785                            DIR_SEPARATOR_STR,
 786                            filename,
 787                            GNUNET_FS_DIRECTORY_EXT);
 788         }
 789       else
 790         {
 791           GNUNET_asprintf (&full_name,
 792                            "%s%s%s",
 793                            dn,
 794                            DIR_SEPARATOR_STR,
 795                            filename);
 796         }
 797       GNUNET_free (dn);
 798     }
 799   if ( (full_name != NULL) &&
 800        (GNUNET_OK !=
 801         GNUNET_DISK_directory_create_for_file (full_name)) )
 802     {
 803       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 804                   _("Failed to create directory for recursive download of `%s'\n"),
 805                   full_name);
 806       GNUNET_free (full_name);
 807       GNUNET_free_non_null (fn);
 808       return;
 809     }
 810
 811   temp_name = NULL;
 812   if ( (data != NULL) &&
 813        (GNUNET_FS_uri_chk_get_file_size (uri) == length) )
 814     {
 815       if (full_name == NULL)
 816         {
 817           temp_name = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
 818           real_name = temp_name;
 819         }
 820       else
 821         {
 822           real_name = full_name;
 823         }
 824       /* write to disk, then trigger normal download which will instantly progress to completion */
 825       fh = GNUNET_DISK_file_open (real_name,
 826                                   GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE | GNUNET_DISK_OPEN_CREATE,
 827                                   GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
 828       if (fh == NULL)
 829         {
 830           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 831                                     "open",
 832                                     real_name);
 833           GNUNET_free (full_name);
 834           GNUNET_free_non_null (fn);
 835           return;
 836         }
 837       if (length !=
 838           GNUNET_DISK_file_write (fh,
 839                                   data,
 840                                   length))
 841         {
 842           GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
 843                                     "write",
 844                                     full_name);
 845         }
 846       GNUNET_DISK_file_close (fh);
 847     }
 848   GNUNET_FS_download_start (dc->h,
 849                             uri,
 850                             meta,
 851                             full_name, temp_name,
 852                             0,
 853                             GNUNET_FS_uri_chk_get_file_size (uri),
 854                             dc->anonymity,
 855                             dc->options,
 856                             NULL,
 857                             dc);
 858   GNUNET_free_non_null (full_name);
 859   GNUNET_free_non_null (temp_name);
 860   GNUNET_free_non_null (fn);
 861 }
 862
 863
 864 /**
 865  * Free entries in the map.
 866  *
 867  * @param cls unused (NULL)
 868  * @param key unused
 869  * @param entry entry of type "struct DownloadRequest" which is freed
 870  * @return GNUNET_OK
 871  */
 872 static int
 873 free_entry (void *cls,
 874             const GNUNET_HashCode *key,
 875             void *entry)
 876 {
 877   GNUNET_free (entry);
 878   return GNUNET_OK;
 879 }
 880
 881
 882 /**
 883  * Iterator over entries in the pending requests in the 'active' map for the
 884  * reply that we just got.
 885  *
 886  * @param cls closure (our 'struct ProcessResultClosure')
 887  * @param key query for the given value / request
 888  * @param value value in the hash map (a 'struct DownloadRequest')
 889  * @return GNUNET_YES (we should continue to iterate); unless serious error
 890  */
 891 static int
 892 process_result_with_request (void *cls,
 893                              const GNUNET_HashCode * key,
 894                              void *value)
 895 {
 896   struct ProcessResultClosure *prc = cls;
 897   struct DownloadRequest *sm = value;
 898   struct DownloadRequest *ppos;
 899   struct DownloadRequest *pprev;
 900   struct GNUNET_DISK_FileHandle *fh;
 901   struct GNUNET_FS_DownloadContext *dc = prc->dc;
 902   struct GNUNET_CRYPTO_AesSessionKey skey;
 903   struct GNUNET_CRYPTO_AesInitializationVector iv;
 904   char pt[prc->size];
 905   struct GNUNET_FS_ProgressInfo pi;
 906   uint64_t off;
 907   size_t bs;
 908   size_t app;
 909   int i;
 910   struct ContentHashKey *chk;
 911
 912   fh = NULL;
 913   bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (dc->uri->data.chk.file_length),
 914                                             dc->treedepth,
 915                                             sm->offset,
 916                                             sm->depth);
 917   if (prc->size != bs)
 918     {
 919 #if DEBUG_DOWNLOAD
 920       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 921                   "Internal error or bogus download URI (expected %u bytes, got %u)\n",
 922                   bs,
 923                   prc->size);
 924 #endif
 925       dc->emsg = GNUNET_strdup ("Internal error or bogus download URI");
 926       goto signal_error;
 927     }
 928   GNUNET_assert (GNUNET_YES ==
 929                  GNUNET_CONTAINER_multihashmap_remove (dc->active,
 930                                                        &prc->query,
 931                                                        sm));
 932   /* if this request is on the pending list, remove it! */
 933   pprev = NULL;
 934   ppos = dc->pending;
 935   while (ppos != NULL)
 936     {
 937       if (ppos == sm)
 938         {
 939           if (pprev == NULL)
 940             dc->pending = ppos->next;
 941           else
 942             pprev->next = ppos->next;
 943           break;
 944         }
 945       pprev = ppos;
 946       ppos = ppos->next;
 947     }
 948   GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
 949   if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data,
 950                                        prc->size,
 951                                        &skey,
 952                                        &iv,
 953                                        pt))
 954     {
 955       GNUNET_break (0);
 956       dc->emsg = GNUNET_strdup ("internal error decrypting content");
 957       goto signal_error;
 958     }
 959   off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
 960                              sm->offset,
 961                              sm->depth,
 962                              dc->treedepth);
 963   /* save to disk */
 964   if ( ( GNUNET_YES == prc->do_store) &&
 965        ( (dc->filename != NULL) ||
 966          (is_recursive_download (dc)) ) &&
 967        ( (sm->depth == dc->treedepth) ||
 968          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 969     {
 970       fh = GNUNET_DISK_file_open (dc->filename != NULL
 971                                   ? dc->filename
 972                                   : dc->temp_filename,
 973                                   GNUNET_DISK_OPEN_READWRITE |
 974                                   GNUNET_DISK_OPEN_CREATE,
 975                                   GNUNET_DISK_PERM_USER_READ |
 976                                   GNUNET_DISK_PERM_USER_WRITE |
 977                                   GNUNET_DISK_PERM_GROUP_READ |
 978                                   GNUNET_DISK_PERM_OTHER_READ);
 979     }
 980   if ( (NULL == fh) &&
 981        (GNUNET_YES == prc->do_store) &&
 982        ( (dc->filename != NULL) ||
 983          (is_recursive_download (dc)) ) &&
 984        ( (sm->depth == dc->treedepth) ||
 985          (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)) ) )
 986     {
 987       GNUNET_asprintf (&dc->emsg,
 988                        _("Download failed: could not open file `%s': %s\n"),
 989                        dc->filename,
 990                        STRERROR (errno));
 991       goto signal_error;
 992     }
 993   if (fh != NULL)
 994     {
 995 #if DEBUG_DOWNLOAD
 996       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 997                   "Saving decrypted block to disk at offset %llu\n",
 998                   (unsigned long long) off);
 999 #endif
1000       if ( (off  !=
1001             GNUNET_DISK_file_seek (fh,
1002                                    off,
1003                                    GNUNET_DISK_SEEK_SET) ) )
1004         {
1005           GNUNET_asprintf (&dc->emsg,
1006                            _("Failed to seek to offset %llu in file `%s': %s\n"),
1007                            (unsigned long long) off,
1008                            dc->filename,
1009                            STRERROR (errno));
1010           goto signal_error;
1011         }
1012       if (prc->size !=
1013           GNUNET_DISK_file_write (fh,
1014                                   pt,
1015                                   prc->size))
1016         {
1017           GNUNET_asprintf (&dc->emsg,
1018                            _("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"),
1019                            (unsigned int) prc->size,
1020                            (unsigned long long) off,
1021                            dc->filename,
1022                            STRERROR (errno));
1023           goto signal_error;
1024         }
1025       GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
1026       fh = NULL;
1027     }
1028   if (sm->depth == dc->treedepth)
1029     {
1030       app = prc->size;
1031       if (sm->offset < dc->offset)
1032         {
1033           /* starting offset begins in the middle of pt,
1034              do not count first bytes as progress */
1035           GNUNET_assert (app > (dc->offset - sm->offset));
1036           app -= (dc->offset - sm->offset);
1037         }
1038       if (sm->offset + prc->size > dc->offset + dc->length)
1039         {
1040           /* end of block is after relevant range,
1041              do not count last bytes as progress */
1042           GNUNET_assert (app > (sm->offset + prc->size) - (dc->offset + dc->length));
1043           app -= (sm->offset + prc->size) - (dc->offset + dc->length);
1044         }
1045       dc->completed += app;
1046
1047       /* do recursive download if option is set and either meta data
1048          says it is a directory or if no meta data is given AND filename
1049          ends in '.gnd' (top-level case) */
1050       if (is_recursive_download (dc))
1051         GNUNET_FS_directory_list_contents (prc->size,
1052                                            pt,
1053                                            off,
1054                                            &trigger_recursive_download,
1055                                            dc);
1056
1057     }
1058   pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
1059   pi.value.download.specifics.progress.data = pt;
1060   pi.value.download.specifics.progress.offset = sm->offset;
1061   pi.value.download.specifics.progress.data_len = prc->size;
1062   pi.value.download.specifics.progress.depth = sm->depth;
1063   GNUNET_FS_download_make_status_ (&pi, dc);
1064   GNUNET_assert (dc->completed <= dc->length);
1065   if (dc->completed == dc->length)
1066     {
1067 #if DEBUG_DOWNLOAD
1068       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1069                   "Download completed, truncating file to desired length %llu\n",
1070                   (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1071 #endif
1072       /* truncate file to size (since we store IBlocks at the end) */
1073       if (dc->filename != NULL)
1074         {
1075           if (0 != truncate (dc->filename,
1076                              GNUNET_ntohll (dc->uri->data.chk.file_length)))
1077             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1078                                       "truncate",
1079                                       dc->filename);
1080         }
1081       if (dc->job_queue != NULL)
1082         {
1083           GNUNET_FS_dequeue_ (dc->job_queue);
1084           dc->job_queue = NULL;
1085         }
1086       if (is_recursive_download (dc))
1087         full_recursive_download (dc);
1088       if (dc->child_head == NULL)
1089         {
1090           /* signal completion */
1091           pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
1092           GNUNET_FS_download_make_status_ (&pi, dc);
1093           if (dc->parent != NULL)
1094             check_completed (dc->parent);
1095         }
1096       GNUNET_assert (sm->depth == dc->treedepth);
1097     }
1098   if (sm->depth == dc->treedepth)
1099     {
1100       GNUNET_FS_download_sync_ (dc);
1101       GNUNET_free (sm);
1102       return GNUNET_YES;
1103     }
1104 #if DEBUG_DOWNLOAD
1105   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1106               "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1107               sm->depth,
1108               (unsigned long long) sm->offset);
1109 #endif
1110   GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1111   chk = (struct ContentHashKey*) pt;
1112   for (i=(prc->size / sizeof(struct ContentHashKey))-1;i>=0;i--)
1113     {
1114       off = compute_dblock_offset (sm->offset,
1115                                    sm->depth,
1116                                    dc->treedepth,
1117                                    i);
1118       if ( (off + DBLOCK_SIZE >= dc->offset) &&
1119            (off < dc->offset + dc->length) )
1120         schedule_block_download (dc,
1121                                  &chk[i],
1122                                  off,
1123                                  sm->depth + 1);
1124     }
1125   GNUNET_free (sm);
1126   GNUNET_FS_download_sync_ (dc);
1127   return GNUNET_YES;
1128
1129  signal_error:
1130   if (fh != NULL)
1131     GNUNET_DISK_file_close (fh);
1132   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
1133   pi.value.download.specifics.error.message = dc->emsg;
1134   GNUNET_FS_download_make_status_ (&pi, dc);
1135   /* abort all pending requests */
1136   if (NULL != dc->th)
1137     {
1138       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1139       dc->th = NULL;
1140     }
1141   GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1142   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1143                                          &free_entry,
1144                                          NULL);
1145   dc->pending = NULL;
1146   dc->client = NULL;
1147   GNUNET_free (sm);
1148   GNUNET_FS_download_sync_ (dc);
1149   return GNUNET_NO;
1150 }
1151
1152
1153 /**
1154  * Process a download result.
1155  *
1156  * @param dc our download context
1157  * @param type type of the result
1158  * @param data the (encrypted) response
1159  * @param size size of data
1160  */
1161 static void
1162 process_result (struct GNUNET_FS_DownloadContext *dc,
1163                 enum GNUNET_BLOCK_Type type,
1164                 const void *data,
1165                 size_t size)
1166 {
1167   struct ProcessResultClosure prc;
1168
1169   prc.dc = dc;
1170   prc.data = data;
1171   prc.size = size;
1172   prc.type = type;
1173   prc.do_store = GNUNET_YES;
1174   GNUNET_CRYPTO_hash (data, size, &prc.query);
1175 #if DEBUG_DOWNLOAD
1176   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1177               "Received result for query `%s' from `%s'-service\n",
1178               GNUNET_h2s (&prc.query),
1179               "FS");
1180 #endif
1181   GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
1182                                               &prc.query,
1183                                               &process_result_with_request,
1184                                               &prc);
1185 }
1186
1187
1188 /**
1189  * Type of a function to call when we receive a message
1190  * from the service.
1191  *
1192  * @param cls closure
1193  * @param msg message received, NULL on timeout or fatal error
1194  */
1195 static void
1196 receive_results (void *cls,
1197                  const struct GNUNET_MessageHeader * msg)
1198 {
1199   struct GNUNET_FS_DownloadContext *dc = cls;
1200   const struct PutMessage *cm;
1201   uint16_t msize;
1202
1203   if ( (NULL == msg) ||
1204        (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) ||
1205        (sizeof (struct PutMessage) > ntohs(msg->size)) )
1206     {
1207       GNUNET_break (msg == NULL);
1208       try_reconnect (dc);
1209       return;
1210     }
1211   msize = ntohs(msg->size);
1212   cm = (const struct PutMessage*) msg;
1213   process_result (dc,
1214                   ntohl (cm->type),
1215                   &cm[1],
1216                   msize - sizeof (struct PutMessage));
1217   if (dc->client == NULL)
1218     return; /* fatal error */
1219   /* continue receiving */
1220   GNUNET_CLIENT_receive (dc->client,
1221                          &receive_results,
1222                          dc,
1223                          GNUNET_TIME_UNIT_FOREVER_REL);
1224 }
1225
1226
1227
1228 /**
1229  * We're ready to transmit a search request to the
1230  * file-sharing service.  Do it.  If there is
1231  * more than one request pending, try to send
1232  * multiple or request another transmission.
1233  *
1234  * @param cls closure
1235  * @param size number of bytes available in buf
1236  * @param buf where the callee should write the message
1237  * @return number of bytes written to buf
1238  */
1239 static size_t
1240 transmit_download_request (void *cls,
1241                            size_t size,
1242                            void *buf)
1243 {
1244   struct GNUNET_FS_DownloadContext *dc = cls;
1245   size_t msize;
1246   struct SearchMessage *sm;
1247
1248   dc->th = NULL;
1249   if (NULL == buf)
1250     {
1251 #if DEBUG_DOWNLOAD
1252       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1253                   "Transmitting download request failed, trying to reconnect\n");
1254 #endif
1255       try_reconnect (dc);
1256       return 0;
1257     }
1258   GNUNET_assert (size >= sizeof (struct SearchMessage));
1259   msize = 0;
1260   sm = buf;
1261   while ( (dc->pending != NULL) &&
1262           (size >= msize + sizeof (struct SearchMessage)) )
1263     {
1264 #if DEBUG_DOWNLOAD
1265       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1266                   "Transmitting download request for `%s' to `%s'-service\n",
1267                   GNUNET_h2s (&dc->pending->chk.query),
1268                   "FS");
1269 #endif
1270       memset (sm, 0, sizeof (struct SearchMessage));
1271       sm->header.size = htons (sizeof (struct SearchMessage));
1272       sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
1273       if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
1274         sm->options = htonl (1);
1275       else
1276         sm->options = htonl (0);
1277       if (dc->pending->depth == dc->treedepth)
1278         sm->type = htonl (GNUNET_BLOCK_TYPE_DBLOCK);
1279       else
1280         sm->type = htonl (GNUNET_BLOCK_TYPE_IBLOCK);
1281       sm->anonymity_level = htonl (dc->anonymity);
1282       sm->target = dc->target.hashPubKey;
1283       sm->query = dc->pending->chk.query;
1284       dc->pending->is_pending = GNUNET_NO;
1285       dc->pending = dc->pending->next;
1286       msize += sizeof (struct SearchMessage);
1287       sm++;
1288     }
1289   if (dc->pending != NULL)
1290     dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1291                                                   sizeof (struct SearchMessage),
1292                                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1293                                                   GNUNET_NO,
1294                                                   &transmit_download_request,
1295                                                   dc);
1296   return msize;
1297 }
1298
1299
1300 /**
1301  * Reconnect to the FS service and transmit our queries NOW.
1302  *
1303  * @param cls our download context
1304  * @param tc unused
1305  */
1306 static void
1307 do_reconnect (void *cls,
1308               const struct GNUNET_SCHEDULER_TaskContext *tc)
1309 {
1310   struct GNUNET_FS_DownloadContext *dc = cls;
1311   struct GNUNET_CLIENT_Connection *client;
1312
1313   dc->task = GNUNET_SCHEDULER_NO_TASK;
1314   client = GNUNET_CLIENT_connect (dc->h->sched,
1315                                   "fs",
1316                                   dc->h->cfg);
1317   if (NULL == client)
1318     {
1319       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1320                   "Connecting to `%s'-service failed, will try again.\n",
1321                   "FS");
1322       try_reconnect (dc);
1323       return;
1324     }
1325   dc->client = client;
1326   dc->th = GNUNET_CLIENT_notify_transmit_ready (client,
1327                                                 sizeof (struct SearchMessage),
1328                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1329                                                 GNUNET_NO,
1330                                                 &transmit_download_request,
1331                                                 dc);
1332   GNUNET_CLIENT_receive (client,
1333                          &receive_results,
1334                          dc,
1335                          GNUNET_TIME_UNIT_FOREVER_REL);
1336 }
1337
1338
1339 /**
1340  * Add entries that are not yet pending back to the pending list.
1341  *
1342  * @param cls our download context
1343  * @param key unused
1344  * @param entry entry of type "struct DownloadRequest"
1345  * @return GNUNET_OK
1346  */
1347 static int
1348 retry_entry (void *cls,
1349              const GNUNET_HashCode *key,
1350              void *entry)
1351 {
1352   struct GNUNET_FS_DownloadContext *dc = cls;
1353   struct DownloadRequest *dr = entry;
1354
1355   if (! dr->is_pending)
1356     {
1357       dr->next = dc->pending;
1358       dr->is_pending = GNUNET_YES;
1359       dc->pending = entry;
1360     }
1361   return GNUNET_OK;
1362 }
1363
1364
1365 /**
1366  * We've lost our connection with the FS service.
1367  * Re-establish it and re-transmit all of our
1368  * pending requests.
1369  *
1370  * @param dc download context that is having trouble
1371  */
1372 static void
1373 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
1374 {
1375
1376   if (NULL != dc->client)
1377     {
1378 #if DEBUG_DOWNLOAD
1379       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1380                   "Moving all requests back to pending list\n");
1381 #endif
1382       if (NULL != dc->th)
1383         {
1384           GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1385           dc->th = NULL;
1386         }
1387       GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1388                                              &retry_entry,
1389                                              dc);
1390       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1391       dc->client = NULL;
1392     }
1393 #if DEBUG_DOWNLOAD
1394   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1395               "Will try to reconnect in 1s\n");
1396 #endif
1397   dc->task
1398     = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
1399                                     GNUNET_TIME_UNIT_SECONDS,
1400                                     &do_reconnect,
1401                                     dc);
1402 }
1403
1404
1405
1406 /**
1407  * We're allowed to ask the FS service for our blocks.  Start the download.
1408  *
1409  * @param cls the 'struct GNUNET_FS_DownloadContext'
1410  * @param client handle to use for communcation with FS (we must destroy it!)
1411  */
1412 static void
1413 activate_fs_download (void *cls,
1414                       struct GNUNET_CLIENT_Connection *client)
1415 {
1416   struct GNUNET_FS_DownloadContext *dc = cls;
1417   struct GNUNET_FS_ProgressInfo pi;
1418
1419 #if DEBUG_DOWNLOAD
1420   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1421               "Download activated\n");
1422 #endif
1423   GNUNET_assert (NULL != client);
1424   GNUNET_assert (dc->client == NULL);
1425   GNUNET_assert (dc->th == NULL);
1426   dc->client = client;
1427   GNUNET_CLIENT_receive (client,
1428                          &receive_results,
1429                          dc,
1430                          GNUNET_TIME_UNIT_FOREVER_REL);
1431   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
1432   GNUNET_FS_download_make_status_ (&pi, dc);
1433   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1434                                          &retry_entry,
1435                                          dc);
1436 #if DEBUG_DOWNLOAD
1437   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1438               "Asking for transmission to FS service\n");
1439 #endif
1440   dc->th = GNUNET_CLIENT_notify_transmit_ready (dc->client,
1441                                                 sizeof (struct SearchMessage),
1442                                                 GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1443                                                 GNUNET_NO,
1444                                                 &transmit_download_request,
1445                                                 dc);
1446   GNUNET_assert (dc->th != NULL);
1447 }
1448
1449
1450 /**
1451  * We must stop to ask the FS service for our blocks.  Pause the download.
1452  *
1453  * @param cls the 'struct GNUNET_FS_DownloadContext'
1454  */
1455 static void
1456 deactivate_fs_download (void *cls)
1457 {
1458   struct GNUNET_FS_DownloadContext *dc = cls;
1459   struct GNUNET_FS_ProgressInfo pi;
1460
1461 #if DEBUG_DOWNLOAD
1462   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1463               "Download deactivated\n");
1464 #endif
1465   if (NULL != dc->th)
1466     {
1467       GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th);
1468       dc->th = NULL;
1469     }
1470   if (NULL != dc->client)
1471     {
1472       GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO);
1473       dc->client = NULL;
1474     }
1475   pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
1476   GNUNET_FS_download_make_status_ (&pi, dc);
1477 }
1478
1479
1480 /**
1481  * Create SUSPEND event for the given download operation
1482  * and then clean up our state (without stop signal).
1483  *
1484  * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
1485  */
1486 void
1487 GNUNET_FS_download_signal_suspend_ (void *cls)
1488 {
1489   struct GNUNET_FS_DownloadContext *dc = cls;
1490   struct GNUNET_FS_ProgressInfo pi;
1491
1492   if (dc->top != NULL)
1493     GNUNET_FS_end_top (dc->h, dc->top);
1494   while (NULL != dc->child_head)
1495     GNUNET_FS_download_signal_suspend_ (dc->child_head);
1496   if (dc->search != NULL)
1497     {
1498       dc->search->download = NULL;
1499       dc->search = NULL;
1500     }
1501   if (dc->job_queue != NULL)
1502     {
1503       GNUNET_FS_dequeue_ (dc->job_queue);
1504       dc->job_queue = NULL;
1505     }
1506   if (dc->parent != NULL)
1507     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1508                                  dc->parent->child_tail,
1509                                  dc);
1510   pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
1511   GNUNET_FS_download_make_status_ (&pi, dc);
1512   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1513     GNUNET_SCHEDULER_cancel (dc->h->sched,
1514                              dc->task);
1515   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1516                                          &free_entry,
1517                                          NULL);
1518   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1519   GNUNET_free_non_null (dc->filename);
1520   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1521   GNUNET_FS_uri_destroy (dc->uri);
1522   GNUNET_free_non_null (dc->temp_filename);
1523   GNUNET_free_non_null (dc->serialization);
1524   GNUNET_free (dc);
1525 }
1526
1527
1528 /**
1529  * Download parts of a file.  Note that this will store
1530  * the blocks at the respective offset in the given file.  Also, the
1531  * download is still using the blocking of the underlying FS
1532  * encoding.  As a result, the download may *write* outside of the
1533  * given boundaries (if offset and length do not match the 32k FS
1534  * block boundaries). <p>
1535  *
1536  * This function should be used to focus a download towards a
1537  * particular portion of the file (optimization), not to strictly
1538  * limit the download to exactly those bytes.
1539  *
1540  * @param h handle to the file sharing subsystem
1541  * @param uri the URI of the file (determines what to download); CHK or LOC URI
1542  * @param meta known metadata for the file (can be NULL)
1543  * @param filename where to store the file, maybe NULL (then no file is
1544  *        created on disk and data must be grabbed from the callbacks)
1545  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1546  *        can be NULL (in which case we will pick a name if needed); the temporary file
1547  *        may already exist, in which case we will try to use the data that is there and
1548  *        if it is not what is desired, will overwrite it
1549  * @param offset at what offset should we start the download (typically 0)
1550  * @param length how many bytes should be downloaded starting at offset
1551  * @param anonymity anonymity level to use for the download
1552  * @param options various options
1553  * @param cctx initial value for the client context for this download
1554  * @param parent parent download to associate this download with (use NULL
1555  *        for top-level downloads; useful for manually-triggered recursive downloads)
1556  * @return context that can be used to control this download
1557  */
1558 struct GNUNET_FS_DownloadContext *
1559 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
1560                           const struct GNUNET_FS_Uri *uri,
1561                           const struct GNUNET_CONTAINER_MetaData *meta,
1562                           const char *filename,
1563                           const char *tempname,
1564                           uint64_t offset,
1565                           uint64_t length,
1566                           uint32_t anonymity,
1567                           enum GNUNET_FS_DownloadOptions options,
1568                           void *cctx,
1569                           struct GNUNET_FS_DownloadContext *parent)
1570 {
1571   struct GNUNET_FS_ProgressInfo pi;
1572   struct GNUNET_FS_DownloadContext *dc;
1573
1574   GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
1575   if ( (offset + length < offset) ||
1576        (offset + length > uri->data.chk.file_length) )
1577     {
1578       GNUNET_break (0);
1579       return NULL;
1580     }
1581 #if DEBUG_DOWNLOAD
1582   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1583               "Starting download `%s' of %llu bytes\n",
1584               filename,
1585               (unsigned long long) length);
1586 #endif
1587   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1588   dc->h = h;
1589   dc->parent = parent;
1590   if (parent != NULL)
1591     {
1592       GNUNET_CONTAINER_DLL_insert (parent->child_head,
1593                                    parent->child_tail,
1594                                    dc);
1595     }
1596   dc->uri = GNUNET_FS_uri_dup (uri);
1597   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1598   dc->client_info = cctx;
1599   dc->start_time = GNUNET_TIME_absolute_get ();
1600   if (NULL != filename)
1601     {
1602       dc->filename = GNUNET_strdup (filename);
1603       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1604         GNUNET_DISK_file_size (filename,
1605                                &dc->old_file_size,
1606                                GNUNET_YES);
1607     }
1608   if (GNUNET_FS_uri_test_loc (dc->uri))
1609     GNUNET_assert (GNUNET_OK ==
1610                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1611                                                         &dc->target));
1612   dc->offset = offset;
1613   dc->length = length;
1614   dc->anonymity = anonymity;
1615   dc->options = options;
1616   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1617   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1618   if ( (filename == NULL) &&
1619        (is_recursive_download (dc) ) )
1620     {
1621       if (tempname != NULL)
1622         dc->temp_filename = GNUNET_strdup (tempname);
1623       else
1624         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1625     }
1626
1627 #if DEBUG_DOWNLOAD
1628   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1629               "Download tree has depth %u\n",
1630               dc->treedepth);
1631 #endif
1632   if (parent == NULL)
1633     {
1634       dc->top = GNUNET_FS_make_top (dc->h,
1635                                     &GNUNET_FS_download_signal_suspend_,
1636                                     dc);
1637     }
1638   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1639   pi.value.download.specifics.start.meta = meta;
1640   GNUNET_FS_download_make_status_ (&pi, dc);
1641   schedule_block_download (dc,
1642                            &dc->uri->data.chk.chk,
1643                            0,
1644                            1 /* 0 == CHK, 1 == top */);
1645   GNUNET_FS_download_sync_ (dc);
1646   GNUNET_FS_download_start_downloading_ (dc);
1647   return dc;
1648 }
1649
1650
1651 /**
1652  * Download parts of a file based on a search result.  The download
1653  * will be associated with the search result (and the association
1654  * will be preserved when serializing/deserializing the state).
1655  * If the search is stopped, the download will not be aborted but
1656  * be 'promoted' to a stand-alone download.
1657  *
1658  * As with the other download function, this will store
1659  * the blocks at the respective offset in the given file.  Also, the
1660  * download is still using the blocking of the underlying FS
1661  * encoding.  As a result, the download may *write* outside of the
1662  * given boundaries (if offset and length do not match the 32k FS
1663  * block boundaries). <p>
1664  *
1665  * The given range can be used to focus a download towards a
1666  * particular portion of the file (optimization), not to strictly
1667  * limit the download to exactly those bytes.
1668  *
1669  * @param h handle to the file sharing subsystem
1670  * @param sr the search result to use for the download (determines uri and
1671  *        meta data and associations)
1672  * @param filename where to store the file, maybe NULL (then no file is
1673  *        created on disk and data must be grabbed from the callbacks)
1674  * @param tempname where to store temporary file data, not used if filename is non-NULL;
1675  *        can be NULL (in which case we will pick a name if needed); the temporary file
1676  *        may already exist, in which case we will try to use the data that is there and
1677  *        if it is not what is desired, will overwrite it
1678  * @param offset at what offset should we start the download (typically 0)
1679  * @param length how many bytes should be downloaded starting at offset
1680  * @param anonymity anonymity level to use for the download
1681  * @param options various download options
1682  * @param cctx initial value for the client context for this download
1683  * @return context that can be used to control this download
1684  */
1685 struct GNUNET_FS_DownloadContext *
1686 GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
1687                                       struct GNUNET_FS_SearchResult *sr,
1688                                       const char *filename,
1689                                       const char *tempname,
1690                                       uint64_t offset,
1691                                       uint64_t length,
1692                                       uint32_t anonymity,
1693                                       enum GNUNET_FS_DownloadOptions options,
1694                                       void *cctx)
1695 {
1696   struct GNUNET_FS_ProgressInfo pi;
1697   struct GNUNET_FS_DownloadContext *dc;
1698
1699   if ( (sr == NULL) ||
1700        (sr->download != NULL) )
1701     {
1702       GNUNET_break (0);
1703       return NULL;
1704     }
1705   GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri));
1706   if ( (offset + length < offset) ||
1707        (offset + length > sr->uri->data.chk.file_length) )
1708     {
1709       GNUNET_break (0);
1710       return NULL;
1711     }
1712 #if DEBUG_DOWNLOAD
1713   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1714               "Starting download `%s' of %llu bytes\n",
1715               filename,
1716               (unsigned long long) length);
1717 #endif
1718   dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
1719   dc->h = h;
1720   dc->search = sr;
1721   sr->download = dc;
1722   if (sr->probe_ctx != NULL)
1723     {
1724       GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
1725       sr->probe_ctx = NULL;
1726     }
1727   dc->uri = GNUNET_FS_uri_dup (sr->uri);
1728   dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta);
1729   dc->client_info = cctx;
1730   dc->start_time = GNUNET_TIME_absolute_get ();
1731   if (NULL != filename)
1732     {
1733       dc->filename = GNUNET_strdup (filename);
1734       if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1735         GNUNET_DISK_file_size (filename,
1736                                &dc->old_file_size,
1737                                GNUNET_YES);
1738     }
1739   if (GNUNET_FS_uri_test_loc (dc->uri))
1740     GNUNET_assert (GNUNET_OK ==
1741                    GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
1742                                                         &dc->target));
1743   dc->offset = offset;
1744   dc->length = length;
1745   dc->anonymity = anonymity;
1746   dc->options = options;
1747   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
1748   dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
1749   if ( (filename == NULL) &&
1750        (is_recursive_download (dc) ) )
1751     {
1752       if (tempname != NULL)
1753         dc->temp_filename = GNUNET_strdup (tempname);
1754       else
1755         dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
1756     }
1757
1758 #if DEBUG_DOWNLOAD
1759   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1760               "Download tree has depth %u\n",
1761               dc->treedepth);
1762 #endif
1763   pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
1764   pi.value.download.specifics.start.meta = dc->meta;
1765   GNUNET_FS_download_make_status_ (&pi, dc);
1766   schedule_block_download (dc,
1767                            &dc->uri->data.chk.chk,
1768                            0,
1769                            1 /* 0 == CHK, 1 == top */);
1770   GNUNET_FS_download_sync_ (dc);
1771   GNUNET_FS_download_start_downloading_ (dc);
1772   return dc;
1773 }
1774
1775
1776 /**
1777  * Start the downloading process (by entering the queue).
1778  *
1779  * @param dc our download context
1780  */
1781 void
1782 GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
1783 {
1784   GNUNET_assert (dc->job_queue == NULL);
1785   dc->job_queue = GNUNET_FS_queue_ (dc->h,
1786                                     &activate_fs_download,
1787                                     &deactivate_fs_download,
1788                                     dc,
1789                                     (dc->length + DBLOCK_SIZE-1) / DBLOCK_SIZE);
1790 }
1791
1792
1793 /**
1794  * Stop a download (aborts if download is incomplete).
1795  *
1796  * @param dc handle for the download
1797  * @param do_delete delete files of incomplete downloads
1798  */
1799 void
1800 GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc,
1801                          int do_delete)
1802 {
1803   struct GNUNET_FS_ProgressInfo pi;
1804   int have_children;
1805
1806   if (dc->top != NULL)
1807     GNUNET_FS_end_top (dc->h, dc->top);
1808   if (dc->search != NULL)
1809     {
1810       dc->search->download = NULL;
1811       dc->search = NULL;
1812     }
1813   if (dc->job_queue != NULL)
1814     {
1815       GNUNET_FS_dequeue_ (dc->job_queue);
1816       dc->job_queue = NULL;
1817     }
1818   have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
1819   while (NULL != dc->child_head)
1820     GNUNET_FS_download_stop (dc->child_head,
1821                              do_delete);
1822   if (dc->parent != NULL)
1823     GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
1824                                  dc->parent->child_tail,
1825                                  dc);
1826   if (dc->serialization != NULL)
1827     GNUNET_FS_remove_sync_file_ (dc->h,
1828                                  ( (dc->parent != NULL)  || (dc->search != NULL) )
1829                                  ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1830                                  : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD ,
1831                                  dc->serialization);
1832   if ( (GNUNET_YES == have_children) &&
1833        (dc->parent == NULL) )
1834     GNUNET_FS_remove_sync_dir_ (dc->h,
1835                                 (dc->search != NULL)
1836                                 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
1837                                 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
1838                                 dc->serialization);
1839   pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
1840   GNUNET_FS_download_make_status_ (&pi, dc);
1841   if (GNUNET_SCHEDULER_NO_TASK != dc->task)
1842     GNUNET_SCHEDULER_cancel (dc->h->sched,
1843                              dc->task);
1844   GNUNET_CONTAINER_multihashmap_iterate (dc->active,
1845                                          &free_entry,
1846                                          NULL);
1847   GNUNET_CONTAINER_multihashmap_destroy (dc->active);
1848   if (dc->filename != NULL)
1849     {
1850       if ( (dc->completed != dc->length) &&
1851            (GNUNET_YES == do_delete) )
1852         {
1853           if (0 != UNLINK (dc->filename))
1854             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
1855                                       "unlink",
1856                                       dc->filename);
1857         }
1858       GNUNET_free (dc->filename);
1859     }
1860   GNUNET_CONTAINER_meta_data_destroy (dc->meta);
1861   GNUNET_FS_uri_destroy (dc->uri);
1862   if (NULL != dc->temp_filename)
1863     {
1864       if (0 != UNLINK (dc->temp_filename))
1865         GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1866                                   "unlink",
1867                                   dc->temp_filename);
1868       GNUNET_free (dc->temp_filename);
1869     }
1870   GNUNET_free_non_null (dc->serialization);
1871   GNUNET_free (dc);
1872 }
1873
1874 /* end of fs_download.c */