src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see https://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Context for "ds_put_cont".
  42  */
  43 struct PutContCtx
  44 {
  45   /**
  46    * Current publishing context.
  47    */
  48   struct GNUNET_FS_PublishContext *sc;
  49
  50   /**
  51    * Specific file with the block.
  52    */
  53   struct GNUNET_FS_FileInformation *p;
  54
  55   /**
  56    * Function to run next, if any (can be NULL).
  57    */
  58   GNUNET_SCHEDULER_Task cont;
  59
  60   /**
  61    * Closure for cont.
  62    */
  63   void *cont_cls;
  64 };
  65
  66
  67 /**
  68  * Fill in all of the generic fields for
  69  * a publish event and call the callback.
  70  *
  71  * @param pi structure to fill in
  72  * @param sc overall publishing context
  73  * @param p file information for the file being published
  74  * @param offset where in the file are we so far
  75  * @return value returned from callback
  76  */
  77 void *
  78 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  79                                 struct GNUNET_FS_PublishContext *sc,
  80                                 const struct GNUNET_FS_FileInformation *p,
  81                                 uint64_t offset)
  82 {
  83   pi->value.publish.pc = sc;
  84   pi->value.publish.fi = p;
  85   pi->value.publish.cctx
  86     = p->client_info;
  87   pi->value.publish.pctx
  88     = (NULL == p->dir) ? NULL : p->dir->client_info;
  89   pi->value.publish.filename = p->filename;
  90   pi->value.publish.size
  91     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  92   pi->value.publish.eta
  93     = GNUNET_TIME_calculate_eta (p->start_time,
  94                                  offset,
  95                                  pi->value.publish.size);
  96   pi->value.publish.completed = offset;
  97   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  98   pi->value.publish.anonymity = p->anonymity;
  99   return sc->h->upcb (sc->h->upcb_cls,
 100                       pi);
 101 }
 102
 103
 104 /**
 105  * Cleanup the publish context, we're done with it.
 106  *
 107  * @param cls struct to clean up after
 108  * @param tc scheduler context
 109  */
 110 static void
 111 publish_cleanup (void *cls,
 112                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 113 {
 114   struct GNUNET_FS_PublishContext *pc = cls;
 115
 116   if (pc->fhc != NULL)
 117     {
 118       GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
 119       pc->fhc = NULL;
 120     }
 121   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 122   if (pc->namespace != NULL)
 123     GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 124   GNUNET_free_non_null (pc->nid);
 125   GNUNET_free_non_null (pc->nuid);
 126   GNUNET_free_non_null (pc->serialization);
 127   if (pc->dsh != NULL)
 128     {
 129       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 130       pc->dsh = NULL;
 131     }
 132   if (pc->client != NULL)
 133     GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 134   GNUNET_free (pc);
 135 }
 136
 137
 138 /**
 139  * Function called by the datastore API with
 140  * the result from the PUT request.
 141  *
 142  * @param cls our closure
 143  * @param success GNUNET_OK on success
 144  * @param msg error message (or NULL)
 145  */
 146 static void
 147 ds_put_cont (void *cls,
 148              int success,
 149              const char *msg)
 150 {
 151   struct PutContCtx *pcc = cls;
 152   struct GNUNET_FS_ProgressInfo pi;
 153
 154   if (GNUNET_SYSERR == pcc->sc->in_network_wait)
 155     {
 156       /* we were aborted in the meantime, finish shutdown! */
 157       GNUNET_SCHEDULER_add_continuation (&publish_cleanup,
 158                                          pcc->sc,
 159                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 160       GNUNET_free (pcc);
 161       return;
 162     }
 163   GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait);
 164   pcc->sc->in_network_wait = GNUNET_NO;
 165   if (GNUNET_OK != success)
 166     {
 167       GNUNET_asprintf (&pcc->p->emsg,
 168                        _("Publishing failed: %s"),
 169                        msg);
 170       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 171       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 172       pi.value.publish.specifics.error.message = pcc->p->emsg;
 173       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0);
 174       if ( (pcc->p->is_directory == GNUNET_NO) &&
 175            (pcc->p->filename != NULL) &&
 176            (pcc->p->data.file.do_index == GNUNET_YES) )
 177         {
 178           /* run unindex to clean up */
 179           GNUNET_FS_unindex_start (pcc->sc->h,
 180                                    pcc->p->filename,
 181                                    NULL);
 182         }
 183     }
 184   if (NULL != pcc->cont)
 185     pcc->sc->upload_task
 186       = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 187                                             pcc->cont,
 188                                             pcc->cont_cls);
 189   GNUNET_free (pcc);
 190 }
 191
 192
 193 /**
 194  * Generate the callback that signals clients
 195  * that a file (or directory) has been completely
 196  * published.
 197  *
 198  * @param p the completed upload
 199  * @param sc context of the publication
 200  */
 201 static void
 202 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 203                            struct GNUNET_FS_PublishContext *sc)
 204 {
 205   struct GNUNET_FS_ProgressInfo pi;
 206
 207   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 208   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 209   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 210   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 211                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 212 }
 213
 214
 215 /**
 216  * Generate the callback that signals clients
 217  * that a file (or directory) has encountered
 218  * a problem during publication.
 219  *
 220  * @param p the upload that had trouble
 221  * @param sc context of the publication
 222  * @param emsg error message
 223  */
 224 static void
 225 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 226                       struct GNUNET_FS_PublishContext *sc,
 227                       const char *emsg)
 228 {
 229   struct GNUNET_FS_ProgressInfo pi;
 230
 231   p->emsg = GNUNET_strdup (emsg);
 232   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 233   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 234   pi.value.publish.specifics.error.message =emsg;
 235   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 236   if ( (p->is_directory == GNUNET_NO) &&
 237        (p->filename != NULL) &&
 238        (p->data.file.do_index == GNUNET_YES) )
 239     {
 240       /* run unindex to clean up */
 241       GNUNET_FS_unindex_start (sc->h,
 242                                p->filename,
 243                                NULL);
 244     }
 245
 246 }
 247
 248
 249 /**
 250  * Datastore returns from reservation cancel request.
 251  *
 252  * @param cls the 'struct GNUNET_FS_PublishContext'
 253  * @param success success code (not used)
 254  * @param msg error message (typically NULL, not used)
 255  */
 256 static void
 257 finish_release_reserve (void *cls,
 258                         int success,
 259                         const char *msg)
 260 {
 261   struct GNUNET_FS_PublishContext *pc = cls;
 262
 263   pc->qre = NULL;
 264   signal_publish_completion (pc->fi, pc);
 265   pc->all_done = GNUNET_YES;
 266   GNUNET_FS_publish_sync_ (pc);
 267 }
 268
 269
 270 /**
 271  * We've finished publishing the SBlock as part of a larger upload.
 272  * Check the result and complete the larger upload.
 273  *
 274  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 275  * @param uri URI of the published SBlock
 276  * @param emsg NULL on success, otherwise error message
 277  */
 278 static void
 279 publish_sblocks_cont (void *cls,
 280                       const struct GNUNET_FS_Uri *uri,
 281                       const char *emsg)
 282 {
 283   struct GNUNET_FS_PublishContext *pc = cls;
 284   if (NULL != emsg)
 285     {
 286       signal_publish_error (pc->fi,
 287                             pc,
 288                             emsg);
 289       GNUNET_FS_publish_sync_ (pc);
 290       return;
 291     }
 292   GNUNET_assert (pc->qre == NULL);
 293   if ( (pc->dsh != NULL) &&
 294        (pc->rid != 0) )
 295     {
 296       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 297                                                   pc->rid,
 298                                                   UINT_MAX,
 299                                                   UINT_MAX,
 300                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 301                                                   &finish_release_reserve,
 302                                                   pc);
 303     }
 304   else
 305     {
 306       finish_release_reserve (pc, GNUNET_OK, NULL);
 307     }
 308 }
 309
 310
 311 /**
 312  * We are almost done publishing the structure,
 313  * add SBlocks (if needed).
 314  *
 315  * @param sc overall upload data
 316  */
 317 static void
 318 publish_sblock (struct GNUNET_FS_PublishContext *sc)
 319 {
 320   if (NULL != sc->namespace)
 321     GNUNET_FS_publish_sks (sc->h,
 322                            sc->namespace,
 323                            sc->nid,
 324                            sc->nuid,
 325                            sc->fi->meta,
 326                            sc->fi->chk_uri,
 327                            sc->fi->expirationTime,
 328                            sc->fi->anonymity,
 329                            sc->fi->priority,
 330                            sc->options,
 331                            &publish_sblocks_cont,
 332                            sc);
 333   else
 334     publish_sblocks_cont (sc, NULL, NULL);
 335 }
 336
 337
 338 /**
 339  * We've finished publishing a KBlock as part of a larger upload.
 340  * Check the result and continue the larger upload.
 341  *
 342  * @param cls the "struct GNUNET_FS_PublishContext*"
 343  *        of the larger upload
 344  * @param uri URI of the published blocks
 345  * @param emsg NULL on success, otherwise error message
 346  */
 347 static void
 348 publish_kblocks_cont (void *cls,
 349                       const struct GNUNET_FS_Uri *uri,
 350                       const char *emsg)
 351 {
 352   struct GNUNET_FS_PublishContext *pc = cls;
 353   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 354
 355   if (NULL != emsg)
 356     {
 357 #if DEBUG_PUBLISH
 358       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 359                   "Error uploading KSK blocks: %s\n",
 360                   emsg);
 361 #endif
 362       signal_publish_error (p, pc, emsg);
 363       GNUNET_FS_file_information_sync_ (p);
 364       GNUNET_FS_publish_sync_ (pc);
 365       pc->upload_task
 366         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 367                                               &GNUNET_FS_publish_main_,
 368                                               pc);
 369       return;
 370     }
 371 #if DEBUG_PUBLISH
 372   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 373               "KSK blocks published, moving on to next file\n");
 374 #endif
 375   if (NULL != p->dir)
 376     signal_publish_completion (p, pc);
 377   /* move on to next file */
 378   if (NULL != p->next)
 379     pc->fi_pos = p->next;
 380   else
 381     pc->fi_pos = p->dir;
 382   GNUNET_FS_publish_sync_ (pc);
 383   pc->upload_task
 384     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 385                                           &GNUNET_FS_publish_main_,
 386                                           pc);
 387 }
 388
 389
 390 /**
 391  * Function called by the tree encoder to obtain
 392  * a block of plaintext data (for the lowest level
 393  * of the tree).
 394  *
 395  * @param cls our publishing context
 396  * @param offset identifies which block to get
 397  * @param max (maximum) number of bytes to get; returning
 398  *        fewer will also cause errors
 399  * @param buf where to copy the plaintext buffer
 400  * @param emsg location to store an error message (on error)
 401  * @return number of bytes copied to buf, 0 on error
 402  */
 403 static size_t
 404 block_reader (void *cls,
 405               uint64_t offset,
 406               size_t max,
 407               void *buf,
 408               char **emsg)
 409 {
 410   struct GNUNET_FS_PublishContext *sc = cls;
 411   struct GNUNET_FS_FileInformation *p;
 412   size_t pt_size;
 413   const char *dd;
 414
 415   p = sc->fi_pos;
 416   if (p->is_directory)
 417     {
 418       pt_size = GNUNET_MIN(max,
 419                            p->data.dir.dir_size - offset);
 420       dd = p->data.dir.dir_data;
 421       memcpy (buf,
 422               &dd[offset],
 423               pt_size);
 424     }
 425   else
 426     {
 427       pt_size = GNUNET_MIN(max,
 428                            p->data.file.file_size - offset);
 429       if (pt_size == 0)
 430         return 0; /* calling reader with pt_size==0
 431                      might free buf, so don't! */
 432       if (pt_size !=
 433           p->data.file.reader (p->data.file.reader_cls,
 434                                offset,
 435                                pt_size,
 436                                buf,
 437                                emsg))
 438         return 0;
 439     }
 440   return pt_size;
 441 }
 442
 443
 444 /**
 445  * The tree encoder has finished processing a
 446  * file.   Call it's finish method and deal with
 447  * the final result.
 448  *
 449  * @param cls our publishing context
 450  * @param tc scheduler's task context (not used)
 451  */
 452 static void
 453 encode_cont (void *cls,
 454              const struct GNUNET_SCHEDULER_TaskContext *tc)
 455 {
 456   struct GNUNET_FS_PublishContext *sc = cls;
 457   struct GNUNET_FS_FileInformation *p;
 458   struct GNUNET_FS_ProgressInfo pi;
 459   char *emsg;
 460   uint64_t flen;
 461
 462   p = sc->fi_pos;
 463   GNUNET_FS_tree_encoder_finish (p->te,
 464                                  &p->chk_uri,
 465                                  &emsg);
 466   p->te = NULL;
 467   if (NULL != emsg)
 468     {
 469 #if DEBUG_PUBLISH
 470       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 471                   "Error during tree walk: %s\n",
 472                   emsg);
 473 #endif
 474       GNUNET_asprintf (&p->emsg,
 475                        _("Publishing failed: %s"),
 476                        emsg);
 477       GNUNET_free (emsg);
 478       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 479       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 480       pi.value.publish.specifics.error.message = p->emsg;
 481       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 482     }
 483 #if DEBUG_PUBLISH
 484   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 485               "Finished with tree encoder\n");
 486 #endif
 487   /* final progress event */
 488   flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
 489   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 490   pi.value.publish.specifics.progress.data = NULL;
 491   pi.value.publish.specifics.progress.offset = flen;
 492   pi.value.publish.specifics.progress.data_len = 0;
 493   pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
 494   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, flen);
 495
 496   /* continue with main */
 497   sc->upload_task
 498     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 499                                           &GNUNET_FS_publish_main_,
 500                                           sc);
 501 }
 502
 503
 504 /**
 505  * Function called asking for the current (encoded)
 506  * block to be processed.  After processing the
 507  * client should either call "GNUNET_FS_tree_encode_next"
 508  * or (on error) "GNUNET_FS_tree_encode_finish".
 509  *
 510  * @param cls closure
 511  * @param chk content hash key for the block
 512  * @param offset offset of the block in the file
 513  * @param depth depth of the block in the file, 0 for DBLOCK
 514  * @param type type of the block (IBLOCK or DBLOCK)
 515  * @param block the (encrypted) block
 516  * @param block_size size of block (in bytes)
 517  */
 518 static void
 519 block_proc (void *cls,
 520             const struct ContentHashKey *chk,
 521             uint64_t offset,
 522             unsigned int depth,
 523             enum GNUNET_BLOCK_Type type,
 524             const void *block,
 525             uint16_t block_size)
 526 {
 527   struct GNUNET_FS_PublishContext *sc = cls;
 528   struct GNUNET_FS_FileInformation *p;
 529   struct PutContCtx * dpc_cls;
 530   struct OnDemandBlock odb;
 531
 532   p = sc->fi_pos;
 533   if (NULL == sc->dsh)
 534     {
 535 #if DEBUG_PUBLISH
 536       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 537                   "Waiting for datastore connection\n");
 538 #endif
 539       sc->upload_task
 540         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 541                                               &GNUNET_FS_publish_main_,
 542                                               sc);
 543       return;
 544     }
 545
 546   GNUNET_assert (GNUNET_NO == sc->in_network_wait);
 547   sc->in_network_wait = GNUNET_YES;
 548   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 549   dpc_cls->cont = &GNUNET_FS_publish_main_;
 550   dpc_cls->cont_cls = sc;
 551   dpc_cls->sc = sc;
 552   dpc_cls->p = p;
 553   if ( (! p->is_directory) &&
 554        (GNUNET_YES == p->data.file.do_index) &&
 555        (type == GNUNET_BLOCK_TYPE_FS_DBLOCK) )
 556     {
 557 #if DEBUG_PUBLISH
 558       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 559                   "Indexing block `%s' for offset %llu with index size %u\n",
 560                   GNUNET_h2s (&chk->query),
 561                   (unsigned long long) offset,
 562                   sizeof (struct OnDemandBlock));
 563 #endif
 564       odb.offset = GNUNET_htonll (offset);
 565       odb.file_id = p->data.file.file_id;
 566       GNUNET_DATASTORE_put (sc->dsh,
 567                             (p->is_directory) ? 0 : sc->rid,
 568                             &chk->query,
 569                             sizeof (struct OnDemandBlock),
 570                             &odb,
 571                             GNUNET_BLOCK_TYPE_FS_ONDEMAND,
 572                             p->priority,
 573                             p->anonymity,
 574                             p->expirationTime,
 575                             -2, 1,
 576                             GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 577                             &ds_put_cont,
 578                             dpc_cls);
 579       return;
 580     }
 581 #if DEBUG_PUBLISH
 582   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 583               "Publishing block `%s' for offset %llu with size %u\n",
 584               GNUNET_h2s (&chk->query),
 585               (unsigned long long) offset,
 586               (unsigned int) block_size);
 587 #endif
 588   GNUNET_DATASTORE_put (sc->dsh,
 589                         (p->is_directory) ? 0 : sc->rid,
 590                         &chk->query,
 591                         block_size,
 592                         block,
 593                         type,
 594                         p->priority,
 595                         p->anonymity,
 596                         p->expirationTime,
 597                         -2, 1,
 598                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 599                         &ds_put_cont,
 600                         dpc_cls);
 601 }
 602
 603
 604 /**
 605  * Function called with information about our
 606  * progress in computing the tree encoding.
 607  *
 608  * @param cls closure
 609  * @param offset where are we in the file
 610  * @param pt_block plaintext of the currently processed block
 611  * @param pt_size size of pt_block
 612  * @param depth depth of the block in the tree, 0 for DBLOCK
 613  */
 614 static void
 615 progress_proc (void *cls,
 616                uint64_t offset,
 617                const void *pt_block,
 618                size_t pt_size,
 619                unsigned int depth)
 620 {
 621   struct GNUNET_FS_PublishContext *sc = cls;
 622   struct GNUNET_FS_FileInformation *p;
 623   struct GNUNET_FS_ProgressInfo pi;
 624
 625   p = sc->fi_pos;
 626   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 627   pi.value.publish.specifics.progress.data = pt_block;
 628   pi.value.publish.specifics.progress.offset = offset;
 629   pi.value.publish.specifics.progress.data_len = pt_size;
 630   pi.value.publish.specifics.progress.depth = depth;
 631   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset);
 632 }
 633
 634
 635 /**
 636  * We are uploading a file or directory; load (if necessary) the next
 637  * block into memory, encrypt it and send it to the FS service.  Then
 638  * continue with the main task.
 639  *
 640  * @param sc overall upload data
 641  */
 642 static void
 643 publish_content (struct GNUNET_FS_PublishContext *sc)
 644 {
 645   struct GNUNET_FS_FileInformation *p;
 646   char *emsg;
 647   struct GNUNET_FS_DirectoryBuilder *db;
 648   struct GNUNET_FS_FileInformation *dirpos;
 649   void *raw_data;
 650   uint64_t size;
 651
 652   p = sc->fi_pos;
 653   GNUNET_assert (p != NULL);
 654   if (NULL == p->te)
 655     {
 656       if (p->is_directory)
 657         {
 658 #if DEBUG_PUBLISH
 659           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 660                       "Creating directory\n");
 661 #endif
 662           db = GNUNET_FS_directory_builder_create (p->meta);
 663           dirpos = p->data.dir.entries;
 664           while (NULL != dirpos)
 665             {
 666               if (dirpos->is_directory)
 667                 {
 668                   raw_data = dirpos->data.dir.dir_data;
 669                   dirpos->data.dir.dir_data = NULL;
 670                 }
 671               else
 672                 {
 673                   raw_data = NULL;
 674                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 675                        (dirpos->data.file.file_size > 0) )
 676                     {
 677                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 678                       emsg = NULL;
 679                       if (dirpos->data.file.file_size !=
 680                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 681                                                     0,
 682                                                     dirpos->data.file.file_size,
 683                                                     raw_data,
 684                                                     &emsg))
 685                         {
 686                           GNUNET_free_non_null (emsg);
 687                           GNUNET_free (raw_data);
 688                           raw_data = NULL;
 689                         }
 690                     }
 691                 }
 692               GNUNET_FS_directory_builder_add (db,
 693                                                dirpos->chk_uri,
 694                                                dirpos->meta,
 695                                                raw_data);
 696               GNUNET_free_non_null (raw_data);
 697               dirpos = dirpos->next;
 698             }
 699           GNUNET_FS_directory_builder_finish (db,
 700                                               &p->data.dir.dir_size,
 701                                               &p->data.dir.dir_data);
 702           GNUNET_FS_file_information_sync_ (p);
 703         }
 704       size = (p->is_directory)
 705         ? p->data.dir.dir_size
 706         : p->data.file.file_size;
 707 #if DEBUG_PUBLISH
 708       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 709                   "Creating tree encoder\n");
 710 #endif
 711       p->te = GNUNET_FS_tree_encoder_create (sc->h,
 712                                              size,
 713                                              sc,
 714                                              &block_reader,
 715                                              &block_proc,
 716                                              &progress_proc,
 717                                              &encode_cont);
 718
 719     }
 720 #if DEBUG_PUBLISH
 721   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 722               "Processing next block from tree\n");
 723 #endif
 724   GNUNET_FS_tree_encoder_next (p->te);
 725 }
 726
 727
 728 /**
 729  * Process the response (or lack thereof) from
 730  * the "fs" service to our 'start index' request.
 731  *
 732  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 733  * @param msg the response we got
 734  */
 735 static void
 736 process_index_start_response (void *cls,
 737                               const struct GNUNET_MessageHeader *msg)
 738 {
 739   struct GNUNET_FS_PublishContext *sc = cls;
 740   struct GNUNET_FS_FileInformation *p;
 741   const char *emsg;
 742   uint16_t msize;
 743
 744   GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO);
 745   sc->client = NULL;
 746   p = sc->fi_pos;
 747   if (msg == NULL)
 748     {
 749       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 750                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 751                   p->filename,
 752                   _("timeout on index-start request to `fs' service"));
 753       p->data.file.do_index = GNUNET_NO;
 754       GNUNET_FS_file_information_sync_ (p);
 755       publish_content (sc);
 756       return;
 757     }
 758   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 759     {
 760       msize = ntohs (msg->size);
 761       emsg = (const char *) &msg[1];
 762       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 763            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 764         emsg = gettext_noop ("unknown error");
 765       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 766                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 767                   p->filename,
 768                   gettext (emsg));
 769       p->data.file.do_index = GNUNET_NO;
 770       GNUNET_FS_file_information_sync_ (p);
 771       publish_content (sc);
 772       return;
 773     }
 774   p->data.file.index_start_confirmed = GNUNET_YES;
 775   /* success! continue with indexing */
 776   GNUNET_FS_file_information_sync_ (p);
 777   publish_content (sc);
 778 }
 779
 780
 781 /**
 782  * Function called once the hash computation over an
 783  * indexed file has completed.
 784  *
 785  * @param cls closure, our publishing context
 786  * @param res resulting hash, NULL on error
 787  */
 788 static void
 789 hash_for_index_cb (void *cls,
 790                    const GNUNET_HashCode *
 791                    res)
 792 {
 793   struct GNUNET_FS_PublishContext *sc = cls;
 794   struct GNUNET_FS_FileInformation *p;
 795   struct IndexStartMessage *ism;
 796   size_t slen;
 797   struct GNUNET_CLIENT_Connection *client;
 798   uint64_t dev;
 799   uint64_t ino;
 800   char *fn;
 801
 802   sc->fhc = NULL;
 803   p = sc->fi_pos;
 804   if (NULL == res)
 805     {
 806       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 807                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 808                   p->filename,
 809                   _("failed to compute hash"));
 810       p->data.file.do_index = GNUNET_NO;
 811       GNUNET_FS_file_information_sync_ (p);
 812       publish_content (sc);
 813       return;
 814     }
 815   if (GNUNET_YES == p->data.file.index_start_confirmed)
 816     {
 817       publish_content (sc);
 818       return;
 819     }
 820   fn = GNUNET_STRINGS_filename_expand (p->filename);
 821   GNUNET_assert (fn != NULL);
 822   slen = strlen (fn) + 1;
 823   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 824     {
 825       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 826                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 827                   fn,
 828                   _("filename too long"));
 829       GNUNET_free (fn);
 830       p->data.file.do_index = GNUNET_NO;
 831       GNUNET_FS_file_information_sync_ (p);
 832       publish_content (sc);
 833       return;
 834     }
 835 #if DEBUG_PUBLISH
 836   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 837               "Hash of indexed file `%s' is `%s'\n",
 838               p->filename,
 839               GNUNET_h2s (res));
 840 #endif
 841   if (0 != (sc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
 842     {
 843       p->data.file.file_id = *res;
 844       p->data.file.have_hash = GNUNET_YES;
 845       p->data.file.index_start_confirmed = GNUNET_YES;
 846       GNUNET_FS_file_information_sync_ (p);
 847       publish_content (sc);
 848       GNUNET_free (fn);
 849       return;
 850     }
 851   client = GNUNET_CLIENT_connect ("fs",
 852                                   sc->h->cfg);
 853   if (NULL == client)
 854     {
 855       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 856                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 857                   p->filename,
 858                   _("could not connect to `fs' service"));
 859       p->data.file.do_index = GNUNET_NO;
 860       publish_content (sc);
 861       GNUNET_free (fn);
 862       return;
 863     }
 864   if (p->data.file.have_hash != GNUNET_YES)
 865     {
 866       p->data.file.file_id = *res;
 867       p->data.file.have_hash = GNUNET_YES;
 868       GNUNET_FS_file_information_sync_ (p);
 869     }
 870   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 871                        slen);
 872   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 873                            slen);
 874   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 875   if (GNUNET_OK ==
 876       GNUNET_DISK_file_get_identifiers (p->filename,
 877                                         &dev,
 878                                         &ino))
 879     {
 880       ism->device = GNUNET_htonll (dev);
 881       ism->inode = GNUNET_htonll(ino);
 882     }
 883 #if DEBUG_PUBLISH
 884   else
 885     {
 886       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 887                   _("Failed to get file identifiers for `%s'\n"),
 888                   p->filename);
 889     }
 890 #endif
 891   ism->file_id = *res;
 892   memcpy (&ism[1],
 893           fn,
 894           slen);
 895   GNUNET_free (fn);
 896   sc->client = client;
 897   GNUNET_break (GNUNET_YES ==
 898                 GNUNET_CLIENT_transmit_and_get_response (client,
 899                                                          &ism->header,
 900                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 901                                                          GNUNET_YES,
 902                                                          &process_index_start_response,
 903                                                          sc));
 904   GNUNET_free (ism);
 905 }
 906
 907
 908 /**
 909  * Main function that performs the upload.
 910  *
 911  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 912  * @param tc task context
 913  */
 914 void
 915 GNUNET_FS_publish_main_ (void *cls,
 916                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 917 {
 918   struct GNUNET_FS_PublishContext *pc = cls;
 919   struct GNUNET_FS_ProgressInfo pi;
 920   struct GNUNET_FS_FileInformation *p;
 921   struct GNUNET_FS_Uri *loc;
 922   char *fn;
 923
 924   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 925   p = pc->fi_pos;
 926   if (NULL == p)
 927     {
 928 #if DEBUG_PUBLISH
 929       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 930                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 931 #endif
 932       /* upload of entire hierarchy complete,
 933          publish namespace entries */
 934       GNUNET_FS_publish_sync_ (pc);
 935       publish_sblock (pc);
 936       return;
 937     }
 938   /* find starting position */
 939   while ( (p->is_directory) &&
 940           (NULL != p->data.dir.entries) &&
 941           (NULL == p->emsg) &&
 942           (NULL == p->data.dir.entries->chk_uri) )
 943     {
 944       p = p->data.dir.entries;
 945       pc->fi_pos = p;
 946       GNUNET_FS_publish_sync_ (pc);
 947     }
 948   /* abort on error */
 949   if (NULL != p->emsg)
 950     {
 951 #if DEBUG_PUBLISH
 952       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 953                   "Error uploading: %s\n",
 954                   p->emsg);
 955 #endif
 956       /* error with current file, abort all
 957          related files as well! */
 958       while (NULL != p->dir)
 959         {
 960           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 961                                                        EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
 962           p = p->dir;
 963           if (fn != NULL)
 964             {
 965               GNUNET_asprintf (&p->emsg,
 966                                _("Recursive upload failed at `%s': %s"),
 967                                fn,
 968                                p->emsg);
 969               GNUNET_free (fn);
 970             }
 971           else
 972             {
 973               GNUNET_asprintf (&p->emsg,
 974                                _("Recursive upload failed: %s"),
 975                                p->emsg);
 976             }
 977           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 978           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 979           pi.value.publish.specifics.error.message = p->emsg;
 980           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 981         }
 982       pc->all_done = GNUNET_YES;
 983       GNUNET_FS_publish_sync_ (pc);
 984       return;
 985     }
 986   /* handle completion */
 987   if (NULL != p->chk_uri)
 988     {
 989 #if DEBUG_PUBLISH
 990       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 991                   "File upload complete, now publishing KSK blocks.\n");
 992 #endif
 993       if (0 == p->anonymity)
 994         {
 995           /* zero anonymity, box CHK URI in LOC URI */
 996           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
 997                                           pc->h->cfg,
 998                                           p->expirationTime);
 999           GNUNET_FS_uri_destroy (p->chk_uri);
1000           p->chk_uri = loc;
1001         }
1002       GNUNET_FS_publish_sync_ (pc);
1003       /* upload of "p" complete, publish KBlocks! */
1004       if (p->keywords != NULL)
1005         {
1006           GNUNET_FS_publish_ksk (pc->h,
1007                                  p->keywords,
1008                                  p->meta,
1009                                  p->chk_uri,
1010                                  p->expirationTime,
1011                                  p->anonymity,
1012                                  p->priority,
1013                                  pc->options,
1014                                  &publish_kblocks_cont,
1015                                  pc);
1016         }
1017       else
1018         {
1019           publish_kblocks_cont (pc,
1020                                 p->chk_uri,
1021                                 NULL);
1022         }
1023       return;
1024     }
1025   if ( (!p->is_directory) &&
1026        (p->data.file.do_index) )
1027     {
1028       if (NULL == p->filename)
1029         {
1030           p->data.file.do_index = GNUNET_NO;
1031           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1032                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1033                       "<no-name>",
1034                       _("needs to be an actual file"));
1035           GNUNET_FS_file_information_sync_ (p);
1036           publish_content (pc);
1037           return;
1038         }
1039       if (p->data.file.have_hash)
1040         {
1041           hash_for_index_cb (pc,
1042                              &p->data.file.file_id);
1043         }
1044       else
1045         {
1046           p->start_time = GNUNET_TIME_absolute_get ();
1047           pc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
1048                                              p->filename,
1049                                              HASHING_BLOCKSIZE,
1050                                              &hash_for_index_cb,
1051                                              pc);
1052         }
1053       return;
1054     }
1055   publish_content (pc);
1056 }
1057
1058
1059 /**
1060  * Signal the FS's progress function that we are starting
1061  * an upload.
1062  *
1063  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1064  * @param fi the entry in the publish-structure
1065  * @param length length of the file or directory
1066  * @param meta metadata for the file or directory (can be modified)
1067  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1068  * @param anonymity pointer to selected anonymity level (can be modified)
1069  * @param priority pointer to selected priority (can be modified)
1070  * @param do_index should we index?
1071  * @param expirationTime pointer to selected expiration time (can be modified)
1072  * @param client_info pointer to client context set upon creation (can be modified)
1073  * @return GNUNET_OK to continue (always)
1074  */
1075 static int
1076 fip_signal_start(void *cls,
1077                  struct GNUNET_FS_FileInformation *fi,
1078                  uint64_t length,
1079                  struct GNUNET_CONTAINER_MetaData *meta,
1080                  struct GNUNET_FS_Uri **uri,
1081                  uint32_t *anonymity,
1082                  uint32_t *priority,
1083                  int *do_index,
1084                  struct GNUNET_TIME_Absolute *expirationTime,
1085                  void **client_info)
1086 {
1087   struct GNUNET_FS_PublishContext *sc = cls;
1088   struct GNUNET_FS_ProgressInfo pi;
1089   unsigned int kc;
1090   uint64_t left;
1091
1092   if (*do_index)
1093     {
1094       /* space for on-demand blocks */
1095       sc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1096     }
1097   else
1098     {
1099       /* space for DBlocks */
1100       sc->reserve_space += length;
1101     }
1102   /* entries for IBlocks and DBlocks, space for IBlocks */
1103   left = length;
1104   while (1)
1105     {
1106       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1107       sc->reserve_entries += left;
1108       if (left <= 1)
1109         break;
1110       left = left * sizeof (struct ContentHashKey);
1111       sc->reserve_space += left;
1112     }
1113   sc->reserve_entries++;
1114   /* entries and space for keywords */
1115   if (NULL != *uri)
1116     {
1117       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1118       sc->reserve_entries += kc;
1119       sc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1120     }
1121   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1122   *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
1123   GNUNET_FS_file_information_sync_ (fi);
1124   return GNUNET_OK;
1125 }
1126
1127
1128 /**
1129  * Signal the FS's progress function that we are suspending
1130  * an upload.
1131  *
1132  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1133  * @param fi the entry in the publish-structure
1134  * @param length length of the file or directory
1135  * @param meta metadata for the file or directory (can be modified)
1136  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1137  * @param anonymity pointer to selected anonymity level (can be modified)
1138  * @param priority pointer to selected priority (can be modified)
1139  * @param do_index should we index?
1140  * @param expirationTime pointer to selected expiration time (can be modified)
1141  * @param client_info pointer to client context set upon creation (can be modified)
1142  * @return GNUNET_OK to continue (always)
1143  */
1144 static int
1145 fip_signal_suspend(void *cls,
1146                    struct GNUNET_FS_FileInformation *fi,
1147                    uint64_t length,
1148                    struct GNUNET_CONTAINER_MetaData *meta,
1149                    struct GNUNET_FS_Uri **uri,
1150                    uint32_t *anonymity,
1151                    uint32_t *priority,
1152                    int *do_index,
1153                    struct GNUNET_TIME_Absolute *expirationTime,
1154                    void **client_info)
1155 {
1156   struct GNUNET_FS_PublishContext*sc = cls;
1157   struct GNUNET_FS_ProgressInfo pi;
1158   uint64_t off;
1159
1160   GNUNET_free_non_null (fi->serialization);
1161   fi->serialization = NULL;
1162   off = (fi->chk_uri == NULL) ? 0 : length;
1163   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1164   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1165   *client_info = NULL;
1166   if (NULL != sc->dsh)
1167     {
1168       GNUNET_DATASTORE_disconnect (sc->dsh, GNUNET_NO);
1169       sc->dsh = NULL;
1170     }
1171   return GNUNET_OK;
1172 }
1173
1174
1175 /**
1176  * Create SUSPEND event for the given publish operation
1177  * and then clean up our state (without stop signal).
1178  *
1179  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1180  */
1181 void
1182 GNUNET_FS_publish_signal_suspend_ (void *cls)
1183 {
1184   struct GNUNET_FS_PublishContext *pc = cls;
1185
1186   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1187     {
1188       GNUNET_SCHEDULER_cancel (pc->upload_task);
1189       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1190     }
1191   GNUNET_FS_file_information_inspect (pc->fi,
1192                                       &fip_signal_suspend,
1193                                       pc);
1194   GNUNET_FS_end_top (pc->h, pc->top);
1195   publish_cleanup (pc, NULL);
1196 }
1197
1198
1199 /**
1200  * We have gotten a reply for our space reservation request.
1201  * Either fail (insufficient space) or start publishing for good.
1202  *
1203  * @param cls the 'struct GNUNET_FS_PublishContext*'
1204  * @param success positive reservation ID on success
1205  * @param msg error message on error, otherwise NULL
1206  */
1207 static void
1208 finish_reserve (void *cls,
1209                 int success,
1210                 const char *msg)
1211 {
1212   struct GNUNET_FS_PublishContext *pc = cls;
1213
1214   pc->qre = NULL;
1215   if ( (msg != NULL) ||
1216        (success <= 0) )
1217     {
1218       GNUNET_asprintf (&pc->fi->emsg,
1219                        _("Insufficient space for publishing: %s"),
1220                        msg);
1221       signal_publish_error (pc->fi,
1222                             pc,
1223                             pc->fi->emsg);
1224       return;
1225     }
1226   pc->rid = success;
1227   pc->upload_task
1228     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1229                                           &GNUNET_FS_publish_main_,
1230                                           pc);
1231 }
1232
1233
1234 /**
1235  * Publish a file or directory.
1236  *
1237  * @param h handle to the file sharing subsystem
1238  * @param fi information about the file or directory structure to publish
1239  * @param namespace namespace to publish the file in, NULL for no namespace
1240  * @param nid identifier to use for the publishd content in the namespace
1241  *        (can be NULL, must be NULL if namespace is NULL)
1242  * @param nuid update-identifier that will be used for future updates
1243  *        (can be NULL, must be NULL if namespace or nid is NULL)
1244  * @param options options for the publication
1245  * @return context that can be used to control the publish operation
1246  */
1247 struct GNUNET_FS_PublishContext *
1248 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1249                          struct GNUNET_FS_FileInformation *fi,
1250                          struct GNUNET_FS_Namespace *namespace,
1251                          const char *nid,
1252                          const char *nuid,
1253                          enum GNUNET_FS_PublishOptions options)
1254 {
1255   struct GNUNET_FS_PublishContext *ret;
1256   struct GNUNET_DATASTORE_Handle *dsh;
1257
1258   GNUNET_assert (NULL != h);
1259   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1260     {
1261       dsh = GNUNET_DATASTORE_connect (h->cfg);
1262       if (NULL == dsh)
1263         return NULL;
1264     }
1265   else
1266     {
1267       dsh = NULL;
1268     }
1269   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1270   ret->dsh = dsh;
1271   ret->h = h;
1272   ret->fi = fi;
1273   ret->namespace = namespace;
1274   ret->options = options;
1275   if (namespace != NULL)
1276     {
1277       namespace->rc++;
1278       GNUNET_assert (NULL != nid);
1279       ret->nid = GNUNET_strdup (nid);
1280       if (NULL != nuid)
1281         ret->nuid = GNUNET_strdup (nuid);
1282     }
1283   /* signal start */
1284   GNUNET_FS_file_information_inspect (ret->fi,
1285                                       &fip_signal_start,
1286                                       ret);
1287   ret->fi_pos = ret->fi;
1288   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1289   GNUNET_FS_publish_sync_ (ret);
1290   if (NULL != ret->dsh)
1291     {
1292       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1293                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1294                   (unsigned int) ret->reserve_entries,
1295                   (unsigned long long) ret->reserve_space);
1296       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1297                                            ret->reserve_space,
1298                                            ret->reserve_entries,
1299                                            UINT_MAX,
1300                                            UINT_MAX,
1301                                            GNUNET_TIME_UNIT_FOREVER_REL,
1302                                            &finish_reserve,
1303                                            ret);
1304     }
1305   else
1306     {
1307       ret->upload_task
1308         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1309                                               &GNUNET_FS_publish_main_,
1310                                               ret);
1311     }
1312   return ret;
1313 }
1314
1315
1316 /**
1317  * Signal the FS's progress function that we are stopping
1318  * an upload.
1319  *
1320  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1321  * @param fi the entry in the publish-structure
1322  * @param length length of the file or directory
1323  * @param meta metadata for the file or directory (can be modified)
1324  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1325  * @param anonymity pointer to selected anonymity level (can be modified)
1326  * @param priority pointer to selected priority (can be modified)
1327  * @param do_index should we index?
1328  * @param expirationTime pointer to selected expiration time (can be modified)
1329  * @param client_info pointer to client context set upon creation (can be modified)
1330  * @return GNUNET_OK to continue (always)
1331  */
1332 static int
1333 fip_signal_stop(void *cls,
1334                 struct GNUNET_FS_FileInformation *fi,
1335                 uint64_t length,
1336                 struct GNUNET_CONTAINER_MetaData *meta,
1337                 struct GNUNET_FS_Uri **uri,
1338                 uint32_t *anonymity,
1339                 uint32_t *priority,
1340                 int *do_index,
1341                 struct GNUNET_TIME_Absolute *expirationTime,
1342                 void **client_info)
1343 {
1344   struct GNUNET_FS_PublishContext*sc = cls;
1345   struct GNUNET_FS_ProgressInfo pi;
1346   uint64_t off;
1347
1348   if (fi->serialization != NULL)
1349     {
1350       GNUNET_FS_remove_sync_file_ (sc->h,
1351                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1352                                    fi->serialization);
1353       GNUNET_free (fi->serialization);
1354       fi->serialization = NULL;
1355     }
1356   off = (fi->chk_uri == NULL) ? 0 : length;
1357   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1358   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1359   *client_info = NULL;
1360   return GNUNET_OK;
1361 }
1362
1363
1364 /**
1365  * Stop an upload.  Will abort incomplete uploads (but
1366  * not remove blocks that have already been publishd) or
1367  * simply clean up the state for completed uploads.
1368  * Must NOT be called from within the event callback!
1369  *
1370  * @param pc context for the upload to stop
1371  */
1372 void
1373 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1374 {
1375   GNUNET_FS_end_top (pc->h, pc->top);
1376   if (NULL != pc->qre)
1377     {
1378       GNUNET_DATASTORE_cancel (pc->qre);
1379       pc->qre = NULL;
1380     }
1381   if (NULL != pc->dsh)
1382     {
1383       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1384       pc->dsh = NULL;
1385     }
1386   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1387     {
1388       GNUNET_SCHEDULER_cancel (pc->upload_task);
1389       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1390     }
1391   if (pc->serialization != NULL)
1392     {
1393       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1394       GNUNET_free (pc->serialization);
1395       pc->serialization = NULL;
1396     }
1397   GNUNET_FS_file_information_inspect (pc->fi,
1398                                       &fip_signal_stop,
1399                                       pc);
1400   if (GNUNET_YES == pc->in_network_wait)
1401     {
1402       pc->in_network_wait = GNUNET_SYSERR;
1403       return;
1404     }
1405   publish_cleanup (pc, NULL);
1406 }
1407
1408
1409 /**
1410  * Context for the KSK publication.
1411  */
1412 struct PublishKskContext
1413 {
1414
1415   /**
1416    * Keywords to use.
1417    */
1418   struct GNUNET_FS_Uri *ksk_uri;
1419
1420   /**
1421    * Global FS context.
1422    */
1423   struct GNUNET_FS_Handle *h;
1424
1425   /**
1426    * The master block that we are sending
1427    * (in plaintext), has "mdsize+slen" more
1428    * bytes than the struct would suggest.
1429    */
1430   struct KBlock *kb;
1431
1432   /**
1433    * Buffer of the same size as "kb" for
1434    * the encrypted version.
1435    */
1436   struct KBlock *cpy;
1437
1438   /**
1439    * Handle to the datastore, NULL if we are just
1440    * simulating.
1441    */
1442   struct GNUNET_DATASTORE_Handle *dsh;
1443
1444   /**
1445    * Function to call once we're done.
1446    */
1447   GNUNET_FS_PublishContinuation cont;
1448
1449   /**
1450    * Closure for cont.
1451    */
1452   void *cont_cls;
1453
1454   /**
1455    * When should the KBlocks expire?
1456    */
1457   struct GNUNET_TIME_Absolute expirationTime;
1458
1459   /**
1460    * Size of the serialized metadata.
1461    */
1462   ssize_t mdsize;
1463
1464   /**
1465    * Size of the (CHK) URI as a string.
1466    */
1467   size_t slen;
1468
1469   /**
1470    * Keyword that we are currently processing.
1471    */
1472   unsigned int i;
1473
1474   /**
1475    * Anonymity level for the KBlocks.
1476    */
1477   uint32_t anonymity;
1478
1479   /**
1480    * Priority for the KBlocks.
1481    */
1482   uint32_t priority;
1483 };
1484
1485
1486 /**
1487  * Continuation of "GNUNET_FS_publish_ksk" that performs
1488  * the actual publishing operation (iterating over all
1489  * of the keywords).
1490  *
1491  * @param cls closure of type "struct PublishKskContext*"
1492  * @param tc unused
1493  */
1494 static void
1495 publish_ksk_cont (void *cls,
1496                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1497
1498
1499 /**
1500  * Function called by the datastore API with
1501  * the result from the PUT request.
1502  *
1503  * @param cls closure of type "struct PublishKskContext*"
1504  * @param success GNUNET_OK on success
1505  * @param msg error message (or NULL)
1506  */
1507 static void
1508 kb_put_cont (void *cls,
1509              int success,
1510              const char *msg)
1511 {
1512   struct PublishKskContext *pkc = cls;
1513
1514   if (GNUNET_OK != success)
1515     {
1516       if (NULL != pkc->dsh)
1517         {
1518           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1519           pkc->dsh = NULL;
1520         }
1521       GNUNET_free (pkc->cpy);
1522       GNUNET_free (pkc->kb);
1523       pkc->cont (pkc->cont_cls,
1524                  NULL,
1525                  msg);
1526       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1527       GNUNET_free (pkc);
1528       return;
1529     }
1530   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1531                                      pkc,
1532                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1533 }
1534
1535
1536 /**
1537  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1538  * publishing operation (iterating over all of the keywords).
1539  *
1540  * @param cls closure of type "struct PublishKskContext*"
1541  * @param tc unused
1542  */
1543 static void
1544 publish_ksk_cont (void *cls,
1545                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1546 {
1547   struct PublishKskContext *pkc = cls;
1548   const char *keyword;
1549   GNUNET_HashCode key;
1550   GNUNET_HashCode query;
1551   struct GNUNET_CRYPTO_AesSessionKey skey;
1552   struct GNUNET_CRYPTO_AesInitializationVector iv;
1553   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1554
1555
1556   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1557        (NULL == pkc->dsh) )
1558     {
1559       if (NULL != pkc->dsh)
1560         {
1561           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1562           pkc->dsh = NULL;
1563         }
1564       GNUNET_free (pkc->cpy);
1565       GNUNET_free (pkc->kb);
1566       pkc->cont (pkc->cont_cls,
1567                  pkc->ksk_uri,
1568                  NULL);
1569       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1570       GNUNET_free (pkc);
1571       return;
1572     }
1573   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1574 #if DEBUG_PUBLISH
1575   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1576               "Publishing under keyword `%s'\n",
1577               keyword);
1578 #endif
1579   /* first character of keyword indicates if it is
1580      mandatory or not -- ignore for hashing */
1581   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1582   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1583   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1584                              pkc->slen + pkc->mdsize,
1585                              &skey,
1586                              &iv,
1587                              &pkc->cpy[1]);
1588   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1589   GNUNET_assert (NULL != pk);
1590   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1591   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1592                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1593                       &query);
1594   GNUNET_assert (GNUNET_OK ==
1595                  GNUNET_CRYPTO_rsa_sign (pk,
1596                                          &pkc->cpy->purpose,
1597                                          &pkc->cpy->signature));
1598   GNUNET_CRYPTO_rsa_key_free (pk);
1599   GNUNET_DATASTORE_put (pkc->dsh,
1600                         0,
1601                         &query,
1602                         pkc->mdsize +
1603                         sizeof (struct KBlock) +
1604                         pkc->slen,
1605                         pkc->cpy,
1606                         GNUNET_BLOCK_TYPE_FS_KBLOCK,
1607                         pkc->priority,
1608                         pkc->anonymity,
1609                         pkc->expirationTime,
1610                         -2, 1,
1611                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1612                         &kb_put_cont,
1613                         pkc);
1614 }
1615
1616
1617 /**
1618  * Publish a CHK under various keywords on GNUnet.
1619  *
1620  * @param h handle to the file sharing subsystem
1621  * @param ksk_uri keywords to use
1622  * @param meta metadata to use
1623  * @param uri URI to refer to in the KBlock
1624  * @param expirationTime when the KBlock expires
1625  * @param anonymity anonymity level for the KBlock
1626  * @param priority priority for the KBlock
1627  * @param options publication options
1628  * @param cont continuation
1629  * @param cont_cls closure for cont
1630  */
1631 void
1632 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1633                        const struct GNUNET_FS_Uri *ksk_uri,
1634                        const struct GNUNET_CONTAINER_MetaData *meta,
1635                        const struct GNUNET_FS_Uri *uri,
1636                        struct GNUNET_TIME_Absolute expirationTime,
1637                        uint32_t anonymity,
1638                        uint32_t priority,
1639                        enum GNUNET_FS_PublishOptions options,
1640                        GNUNET_FS_PublishContinuation cont,
1641                        void *cont_cls)
1642 {
1643   struct PublishKskContext *pkc;
1644   char *uris;
1645   size_t size;
1646   char *kbe;
1647   char *sptr;
1648
1649   GNUNET_assert (NULL != uri);
1650   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1651   pkc->h = h;
1652   pkc->expirationTime = expirationTime;
1653   pkc->anonymity = anonymity;
1654   pkc->priority = priority;
1655   pkc->cont = cont;
1656   pkc->cont_cls = cont_cls;
1657   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1658     {
1659       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
1660       if (pkc->dsh == NULL)
1661         {
1662           cont (cont_cls, NULL, _("Could not connect to datastore."));
1663           GNUNET_free (pkc);
1664           return;
1665         }
1666     }
1667   if (meta == NULL)
1668     pkc->mdsize = 0;
1669   else
1670     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1671   GNUNET_assert (pkc->mdsize >= 0);
1672   uris = GNUNET_FS_uri_to_string (uri);
1673   pkc->slen = strlen (uris) + 1;
1674   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1675   if (size > MAX_KBLOCK_SIZE)
1676     {
1677       size = MAX_KBLOCK_SIZE;
1678       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1679     }
1680   pkc->kb = GNUNET_malloc (size);
1681   kbe = (char *) &pkc->kb[1];
1682   memcpy (kbe, uris, pkc->slen);
1683   GNUNET_free (uris);
1684   sptr = &kbe[pkc->slen];
1685   if (meta != NULL)
1686     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1687                                                         &sptr,
1688                                                         pkc->mdsize,
1689                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1690   if (pkc->mdsize == -1)
1691     {
1692       GNUNET_break (0);
1693       GNUNET_free (pkc->kb);
1694       if (pkc->dsh != NULL)
1695         {
1696           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1697           pkc->dsh = NULL;
1698         }
1699       cont (cont_cls, NULL, _("Internal error."));
1700       GNUNET_free (pkc);
1701       return;
1702     }
1703   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1704
1705   pkc->cpy = GNUNET_malloc (size);
1706   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1707                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1708                                   pkc->mdsize +
1709                                   pkc->slen);
1710   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1711   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1712   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1713                                      pkc,
1714                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1715 }
1716
1717
1718 /* end of fs_publish.c */