src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see http://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Context for "ds_put_cont".
  42  */
  43 struct PutContCtx
  44 {
  45   /**
  46    * Current publishing context.
  47    */
  48   struct GNUNET_FS_PublishContext *sc;
  49
  50   /**
  51    * Specific file with the block.
  52    */
  53   struct GNUNET_FS_FileInformation *p;
  54
  55   /**
  56    * Function to run next, if any (can be NULL).
  57    */
  58   GNUNET_SCHEDULER_Task cont;
  59
  60   /**
  61    * Closure for cont.
  62    */
  63   void *cont_cls;
  64 };
  65
  66
  67 /**
  68  * Fill in all of the generic fields for
  69  * a publish event and call the callback.
  70  *
  71  * @param pi structure to fill in
  72  * @param sc overall publishing context
  73  * @param p file information for the file being published
  74  * @param offset where in the file are we so far
  75  * @return value returned from callback
  76  */
  77 void *
  78 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  79                                 struct GNUNET_FS_PublishContext *sc,
  80                                 const struct GNUNET_FS_FileInformation *p,
  81                                 uint64_t offset)
  82 {
  83   pi->value.publish.pc = sc;
  84   pi->value.publish.fi = p;
  85   pi->value.publish.cctx
  86     = p->client_info;
  87   pi->value.publish.pctx
  88     = (NULL == p->dir) ? NULL : p->dir->client_info;
  89   pi->value.publish.filename = p->filename;
  90   pi->value.publish.size
  91     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  92   pi->value.publish.eta
  93     = GNUNET_TIME_calculate_eta (p->start_time,
  94                                  offset,
  95                                  pi->value.publish.size);
  96   pi->value.publish.completed = offset;
  97   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  98   pi->value.publish.anonymity = p->anonymity;
  99   return sc->h->upcb (sc->h->upcb_cls,
 100                       pi);
 101 }
 102
 103
 104 /**
 105  * Cleanup the publish context, we're done with it.
 106  *
 107  * @param cls struct to clean up after
 108  * @param tc scheduler context
 109  */
 110 static void
 111 publish_cleanup (void *cls,
 112                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 113 {
 114   struct GNUNET_FS_PublishContext *pc = cls;
 115
 116   if (pc->fhc != NULL)
 117     {
 118       GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
 119       pc->fhc = NULL;
 120     }
 121   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 122   if (pc->namespace != NULL)
 123     GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 124   GNUNET_free_non_null (pc->nid);
 125   GNUNET_free_non_null (pc->nuid);
 126   GNUNET_free_non_null (pc->serialization);
 127   if (pc->dsh != NULL)
 128     {
 129       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 130       pc->dsh = NULL;
 131     }
 132   if (pc->client != NULL)
 133     GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 134   GNUNET_free (pc);
 135 }
 136
 137
 138 /**
 139  * Function called by the datastore API with
 140  * the result from the PUT request.
 141  *
 142  * @param cls our closure
 143  * @param success GNUNET_OK on success
 144  * @param msg error message (or NULL)
 145  */
 146 static void
 147 ds_put_cont (void *cls,
 148              int success,
 149              const char *msg)
 150 {
 151   struct PutContCtx *pcc = cls;
 152   struct GNUNET_FS_ProgressInfo pi;
 153
 154   if (GNUNET_SYSERR == pcc->sc->in_network_wait)
 155     {
 156       /* we were aborted in the meantime, finish shutdown! */
 157       GNUNET_SCHEDULER_add_continuation (&publish_cleanup,
 158                                          pcc->sc,
 159                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 160       GNUNET_free (pcc);
 161       return;
 162     }
 163   GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait);
 164   pcc->sc->in_network_wait = GNUNET_NO;
 165   if (GNUNET_OK != success)
 166     {
 167       GNUNET_asprintf (&pcc->p->emsg,
 168                        _("Publishing failed: %s"),
 169                        msg);
 170       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 171       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 172       pi.value.publish.specifics.error.message = pcc->p->emsg;
 173       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0);
 174       if ( (pcc->p->is_directory == GNUNET_NO) &&
 175            (pcc->p->filename != NULL) &&
 176            (pcc->p->data.file.do_index == GNUNET_YES) )
 177         {
 178           /* run unindex to clean up */
 179           GNUNET_FS_unindex_start (pcc->sc->h,
 180                                    pcc->p->filename,
 181                                    NULL);
 182         }
 183     }
 184   if (NULL != pcc->cont)
 185     pcc->sc->upload_task
 186       = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 187                                             pcc->cont,
 188                                             pcc->cont_cls);
 189   GNUNET_free (pcc);
 190 }
 191
 192
 193 /**
 194  * Generate the callback that signals clients
 195  * that a file (or directory) has been completely
 196  * published.
 197  *
 198  * @param p the completed upload
 199  * @param sc context of the publication
 200  */
 201 static void
 202 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 203                            struct GNUNET_FS_PublishContext *sc)
 204 {
 205   struct GNUNET_FS_ProgressInfo pi;
 206
 207   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 208   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 209   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 210   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 211                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 212 }
 213
 214
 215 /**
 216  * Generate the callback that signals clients
 217  * that a file (or directory) has encountered
 218  * a problem during publication.
 219  *
 220  * @param p the upload that had trouble
 221  * @param sc context of the publication
 222  * @param emsg error message
 223  */
 224 static void
 225 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 226                       struct GNUNET_FS_PublishContext *sc,
 227                       const char *emsg)
 228 {
 229   struct GNUNET_FS_ProgressInfo pi;
 230
 231   p->emsg = GNUNET_strdup (emsg);
 232   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 233   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 234   pi.value.publish.specifics.error.message =emsg;
 235   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 236   if ( (p->is_directory == GNUNET_NO) &&
 237        (p->filename != NULL) &&
 238        (p->data.file.do_index == GNUNET_YES) )
 239     {
 240       /* run unindex to clean up */
 241       GNUNET_FS_unindex_start (sc->h,
 242                                p->filename,
 243                                NULL);
 244     }
 245
 246 }
 247
 248
 249 /**
 250  * Datastore returns from reservation cancel request.
 251  *
 252  * @param cls the 'struct GNUNET_FS_PublishContext'
 253  * @param success success code (not used)
 254  * @param msg error message (typically NULL, not used)
 255  */
 256 static void
 257 finish_release_reserve (void *cls,
 258                         int success,
 259                         const char *msg)
 260 {
 261   struct GNUNET_FS_PublishContext *pc = cls;
 262
 263   pc->qre = NULL;
 264   signal_publish_completion (pc->fi, pc);
 265   pc->all_done = GNUNET_YES;
 266   GNUNET_FS_publish_sync_ (pc);
 267 }
 268
 269
 270 /**
 271  * We've finished publishing the SBlock as part of a larger upload.
 272  * Check the result and complete the larger upload.
 273  *
 274  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 275  * @param uri URI of the published SBlock
 276  * @param emsg NULL on success, otherwise error message
 277  */
 278 static void
 279 publish_sblocks_cont (void *cls,
 280                       const struct GNUNET_FS_Uri *uri,
 281                       const char *emsg)
 282 {
 283   struct GNUNET_FS_PublishContext *pc = cls;
 284   if (NULL != emsg)
 285     {
 286       signal_publish_error (pc->fi,
 287                             pc,
 288                             emsg);
 289       GNUNET_FS_publish_sync_ (pc);
 290       return;
 291     }
 292   GNUNET_assert (pc->qre == NULL);
 293   if ( (pc->dsh != NULL) &&
 294        (pc->rid != 0) )
 295     {
 296       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 297                                                   pc->rid,
 298                                                   UINT_MAX,
 299                                                   UINT_MAX,
 300                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 301                                                   &finish_release_reserve,
 302                                                   pc);
 303     }
 304   else
 305     {
 306       finish_release_reserve (pc, GNUNET_OK, NULL);
 307     }
 308 }
 309
 310
 311 /**
 312  * We are almost done publishing the structure,
 313  * add SBlocks (if needed).
 314  *
 315  * @param sc overall upload data
 316  */
 317 static void
 318 publish_sblock (struct GNUNET_FS_PublishContext *sc)
 319 {
 320   if (NULL != sc->namespace)
 321     GNUNET_FS_publish_sks (sc->h,
 322                            sc->namespace,
 323                            sc->nid,
 324                            sc->nuid,
 325                            sc->fi->meta,
 326                            sc->fi->chk_uri,
 327                            sc->fi->expirationTime,
 328                            sc->fi->anonymity,
 329                            sc->fi->priority,
 330                            sc->options,
 331                            &publish_sblocks_cont,
 332                            sc);
 333   else
 334     publish_sblocks_cont (sc, NULL, NULL);
 335 }
 336
 337
 338 /**
 339  * We've finished publishing a KBlock as part of a larger upload.
 340  * Check the result and continue the larger upload.
 341  *
 342  * @param cls the "struct GNUNET_FS_PublishContext*"
 343  *        of the larger upload
 344  * @param uri URI of the published blocks
 345  * @param emsg NULL on success, otherwise error message
 346  */
 347 static void
 348 publish_kblocks_cont (void *cls,
 349                       const struct GNUNET_FS_Uri *uri,
 350                       const char *emsg)
 351 {
 352   struct GNUNET_FS_PublishContext *pc = cls;
 353   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 354
 355   if (NULL != emsg)
 356     {
 357 #if DEBUG_PUBLISH
 358       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 359                   "Error uploading KSK blocks: %s\n",
 360                   emsg);
 361 #endif
 362       signal_publish_error (p, pc, emsg);
 363       GNUNET_FS_file_information_sync_ (p);
 364       GNUNET_FS_publish_sync_ (pc);
 365       pc->upload_task
 366         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 367                                               &GNUNET_FS_publish_main_,
 368                                               pc);
 369       return;
 370     }
 371 #if DEBUG_PUBLISH
 372   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 373               "KSK blocks published, moving on to next file\n");
 374 #endif
 375   if (NULL != p->dir)
 376     signal_publish_completion (p, pc);
 377   /* move on to next file */
 378   if (NULL != p->next)
 379     pc->fi_pos = p->next;
 380   else
 381     pc->fi_pos = p->dir;
 382   GNUNET_FS_publish_sync_ (pc);
 383   pc->upload_task
 384     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 385                                           &GNUNET_FS_publish_main_,
 386                                           pc);
 387 }
 388
 389
 390 /**
 391  * Function called by the tree encoder to obtain
 392  * a block of plaintext data (for the lowest level
 393  * of the tree).
 394  *
 395  * @param cls our publishing context
 396  * @param offset identifies which block to get
 397  * @param max (maximum) number of bytes to get; returning
 398  *        fewer will also cause errors
 399  * @param buf where to copy the plaintext buffer
 400  * @param emsg location to store an error message (on error)
 401  * @return number of bytes copied to buf, 0 on error
 402  */
 403 static size_t
 404 block_reader (void *cls,
 405               uint64_t offset,
 406               size_t max,
 407               void *buf,
 408               char **emsg)
 409 {
 410   struct GNUNET_FS_PublishContext *sc = cls;
 411   struct GNUNET_FS_FileInformation *p;
 412   size_t pt_size;
 413   const char *dd;
 414
 415   p = sc->fi_pos;
 416   if (p->is_directory)
 417     {
 418       pt_size = GNUNET_MIN(max,
 419                            p->data.dir.dir_size - offset);
 420       dd = p->data.dir.dir_data;
 421       memcpy (buf,
 422               &dd[offset],
 423               pt_size);
 424     }
 425   else
 426     {
 427       pt_size = GNUNET_MIN(max,
 428                            p->data.file.file_size - offset);
 429       if (pt_size == 0)
 430         return 0; /* calling reader with pt_size==0
 431                      might free buf, so don't! */
 432       if (pt_size !=
 433           p->data.file.reader (p->data.file.reader_cls,
 434                                offset,
 435                                pt_size,
 436                                buf,
 437                                emsg))
 438         return 0;
 439     }
 440   return pt_size;
 441 }
 442
 443
 444 /**
 445  * The tree encoder has finished processing a
 446  * file.   Call it's finish method and deal with
 447  * the final result.
 448  *
 449  * @param cls our publishing context
 450  * @param tc scheduler's task context (not used)
 451  */
 452 static void
 453 encode_cont (void *cls,
 454              const struct GNUNET_SCHEDULER_TaskContext *tc)
 455 {
 456   struct GNUNET_FS_PublishContext *sc = cls;
 457   struct GNUNET_FS_FileInformation *p;
 458   struct GNUNET_FS_ProgressInfo pi;
 459   char *emsg;
 460
 461   p = sc->fi_pos;
 462   GNUNET_FS_tree_encoder_finish (p->te,
 463                                  &p->chk_uri,
 464                                  &emsg);
 465   p->te = NULL;
 466   if (NULL != emsg)
 467     {
 468 #if DEBUG_PUBLISH
 469       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 470                   "Error during tree walk: %s\n",
 471                   emsg);
 472 #endif
 473       GNUNET_asprintf (&p->emsg,
 474                        _("Publishing failed: %s"),
 475                        emsg);
 476       GNUNET_free (emsg);
 477       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 478       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 479       pi.value.publish.specifics.error.message = p->emsg;
 480       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 481     }
 482 #if DEBUG_PUBLISH
 483   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 484               "Finished with tree encoder\n");
 485 #endif
 486   /* final progress event */
 487   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 488   pi.value.publish.specifics.progress.data = NULL;
 489   pi.value.publish.specifics.progress.offset = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
 490   pi.value.publish.specifics.progress.data_len = 0;
 491   pi.value.publish.specifics.progress.depth = 0;
 492   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 493                                                    GNUNET_FS_uri_chk_get_file_size (p->chk_uri));
 494
 495   /* continue with main */
 496   sc->upload_task
 497     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 498                                           &GNUNET_FS_publish_main_,
 499                                           sc);
 500 }
 501
 502
 503 /**
 504  * Function called asking for the current (encoded)
 505  * block to be processed.  After processing the
 506  * client should either call "GNUNET_FS_tree_encode_next"
 507  * or (on error) "GNUNET_FS_tree_encode_finish".
 508  *
 509  * @param cls closure
 510  * @param query the query for the block (key for lookup in the datastore)
 511  * @param offset offset of the block in the file
 512  * @param depth depth of the block in the file
 513  * @param type type of the block (IBLOCK or DBLOCK)
 514  * @param block the (encrypted) block
 515  * @param block_size size of block (in bytes)
 516  */
 517 static void
 518 block_proc (void *cls,
 519             const GNUNET_HashCode *query,
 520             uint64_t offset,
 521             unsigned int depth,
 522             enum GNUNET_BLOCK_Type type,
 523             const void *block,
 524             uint16_t block_size)
 525 {
 526   struct GNUNET_FS_PublishContext *sc = cls;
 527   struct GNUNET_FS_FileInformation *p;
 528   struct PutContCtx * dpc_cls;
 529   struct OnDemandBlock odb;
 530
 531   p = sc->fi_pos;
 532   if (NULL == sc->dsh)
 533     {
 534 #if DEBUG_PUBLISH
 535       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 536                   "Waiting for datastore connection\n");
 537 #endif
 538       sc->upload_task
 539         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 540                                               &GNUNET_FS_publish_main_,
 541                                               sc);
 542       return;
 543     }
 544
 545   GNUNET_assert (GNUNET_NO == sc->in_network_wait);
 546   sc->in_network_wait = GNUNET_YES;
 547   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 548   dpc_cls->cont = &GNUNET_FS_publish_main_;
 549   dpc_cls->cont_cls = sc;
 550   dpc_cls->sc = sc;
 551   dpc_cls->p = p;
 552   if ( (! p->is_directory) &&
 553        (GNUNET_YES == p->data.file.do_index) &&
 554        (type == GNUNET_BLOCK_TYPE_FS_DBLOCK) )
 555     {
 556 #if DEBUG_PUBLISH
 557       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 558                   "Indexing block `%s' for offset %llu with index size %u\n",
 559                   GNUNET_h2s (query),
 560                   (unsigned long long) offset,
 561                   sizeof (struct OnDemandBlock));
 562 #endif
 563       odb.offset = GNUNET_htonll (offset);
 564       odb.file_id = p->data.file.file_id;
 565       GNUNET_DATASTORE_put (sc->dsh,
 566                             (p->is_directory) ? 0 : sc->rid,
 567                             query,
 568                             sizeof (struct OnDemandBlock),
 569                             &odb,
 570                             GNUNET_BLOCK_TYPE_FS_ONDEMAND,
 571                             p->priority,
 572                             p->anonymity,
 573                             p->expirationTime,
 574                             -2, 1,
 575                             GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 576                             &ds_put_cont,
 577                             dpc_cls);
 578       return;
 579     }
 580 #if DEBUG_PUBLISH
 581   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 582               "Publishing block `%s' for offset %llu with size %u\n",
 583               GNUNET_h2s (query),
 584               (unsigned long long) offset,
 585               (unsigned int) block_size);
 586 #endif
 587   GNUNET_DATASTORE_put (sc->dsh,
 588                         (p->is_directory) ? 0 : sc->rid,
 589                         query,
 590                         block_size,
 591                         block,
 592                         type,
 593                         p->priority,
 594                         p->anonymity,
 595                         p->expirationTime,
 596                         -2, 1,
 597                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 598                         &ds_put_cont,
 599                         dpc_cls);
 600 }
 601
 602
 603 /**
 604  * Function called with information about our
 605  * progress in computing the tree encoding.
 606  *
 607  * @param cls closure
 608  * @param offset where are we in the file
 609  * @param pt_block plaintext of the currently processed block
 610  * @param pt_size size of pt_block
 611  * @param depth depth of the block in the tree
 612  */
 613 static void
 614 progress_proc (void *cls,
 615                uint64_t offset,
 616                const void *pt_block,
 617                size_t pt_size,
 618                unsigned int depth)
 619 {
 620   struct GNUNET_FS_PublishContext *sc = cls;
 621   struct GNUNET_FS_FileInformation *p;
 622   struct GNUNET_FS_ProgressInfo pi;
 623
 624   p = sc->fi_pos;
 625   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 626   pi.value.publish.specifics.progress.data = pt_block;
 627   pi.value.publish.specifics.progress.offset = offset;
 628   pi.value.publish.specifics.progress.data_len = pt_size;
 629   pi.value.publish.specifics.progress.depth = depth;
 630   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset);
 631 }
 632
 633
 634 /**
 635  * We are uploading a file or directory; load (if necessary) the next
 636  * block into memory, encrypt it and send it to the FS service.  Then
 637  * continue with the main task.
 638  *
 639  * @param sc overall upload data
 640  */
 641 static void
 642 publish_content (struct GNUNET_FS_PublishContext *sc)
 643 {
 644   struct GNUNET_FS_FileInformation *p;
 645   char *emsg;
 646   struct GNUNET_FS_DirectoryBuilder *db;
 647   struct GNUNET_FS_FileInformation *dirpos;
 648   void *raw_data;
 649   uint64_t size;
 650
 651   p = sc->fi_pos;
 652   GNUNET_assert (p != NULL);
 653   if (NULL == p->te)
 654     {
 655       if (p->is_directory)
 656         {
 657 #if DEBUG_PUBLISH
 658           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 659                       "Creating directory\n");
 660 #endif
 661           db = GNUNET_FS_directory_builder_create (p->meta);
 662           dirpos = p->data.dir.entries;
 663           while (NULL != dirpos)
 664             {
 665               if (dirpos->is_directory)
 666                 {
 667                   raw_data = dirpos->data.dir.dir_data;
 668                   dirpos->data.dir.dir_data = NULL;
 669                 }
 670               else
 671                 {
 672                   raw_data = NULL;
 673                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 674                        (dirpos->data.file.file_size > 0) )
 675                     {
 676                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 677                       emsg = NULL;
 678                       if (dirpos->data.file.file_size !=
 679                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 680                                                     0,
 681                                                     dirpos->data.file.file_size,
 682                                                     raw_data,
 683                                                     &emsg))
 684                         {
 685                           GNUNET_free_non_null (emsg);
 686                           GNUNET_free (raw_data);
 687                           raw_data = NULL;
 688                         }
 689                     }
 690                 }
 691               GNUNET_FS_directory_builder_add (db,
 692                                                dirpos->chk_uri,
 693                                                dirpos->meta,
 694                                                raw_data);
 695               GNUNET_free_non_null (raw_data);
 696               dirpos = dirpos->next;
 697             }
 698           GNUNET_FS_directory_builder_finish (db,
 699                                               &p->data.dir.dir_size,
 700                                               &p->data.dir.dir_data);
 701           GNUNET_FS_file_information_sync_ (p);
 702         }
 703       size = (p->is_directory)
 704         ? p->data.dir.dir_size
 705         : p->data.file.file_size;
 706 #if DEBUG_PUBLISH
 707       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 708                   "Creating tree encoder\n");
 709 #endif
 710       p->te = GNUNET_FS_tree_encoder_create (sc->h,
 711                                              size,
 712                                              sc,
 713                                              &block_reader,
 714                                              &block_proc,
 715                                              &progress_proc,
 716                                              &encode_cont);
 717
 718     }
 719 #if DEBUG_PUBLISH
 720   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 721               "Processing next block from tree\n");
 722 #endif
 723   GNUNET_FS_tree_encoder_next (p->te);
 724 }
 725
 726
 727 /**
 728  * Process the response (or lack thereof) from
 729  * the "fs" service to our 'start index' request.
 730  *
 731  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 732  * @param msg the response we got
 733  */
 734 static void
 735 process_index_start_response (void *cls,
 736                               const struct GNUNET_MessageHeader *msg)
 737 {
 738   struct GNUNET_FS_PublishContext *sc = cls;
 739   struct GNUNET_FS_FileInformation *p;
 740   const char *emsg;
 741   uint16_t msize;
 742
 743   GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO);
 744   sc->client = NULL;
 745   p = sc->fi_pos;
 746   if (msg == NULL)
 747     {
 748       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 749                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 750                   p->filename,
 751                   _("timeout on index-start request to `fs' service"));
 752       p->data.file.do_index = GNUNET_NO;
 753       GNUNET_FS_file_information_sync_ (p);
 754       publish_content (sc);
 755       return;
 756     }
 757   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 758     {
 759       msize = ntohs (msg->size);
 760       emsg = (const char *) &msg[1];
 761       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 762            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 763         emsg = gettext_noop ("unknown error");
 764       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 765                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 766                   p->filename,
 767                   gettext (emsg));
 768       p->data.file.do_index = GNUNET_NO;
 769       GNUNET_FS_file_information_sync_ (p);
 770       publish_content (sc);
 771       return;
 772     }
 773   p->data.file.index_start_confirmed = GNUNET_YES;
 774   /* success! continue with indexing */
 775   GNUNET_FS_file_information_sync_ (p);
 776   publish_content (sc);
 777 }
 778
 779
 780 /**
 781  * Function called once the hash computation over an
 782  * indexed file has completed.
 783  *
 784  * @param cls closure, our publishing context
 785  * @param res resulting hash, NULL on error
 786  */
 787 static void
 788 hash_for_index_cb (void *cls,
 789                    const GNUNET_HashCode *
 790                    res)
 791 {
 792   struct GNUNET_FS_PublishContext *sc = cls;
 793   struct GNUNET_FS_FileInformation *p;
 794   struct IndexStartMessage *ism;
 795   size_t slen;
 796   struct GNUNET_CLIENT_Connection *client;
 797   uint64_t dev;
 798   uint64_t ino;
 799   char *fn;
 800
 801   sc->fhc = NULL;
 802   p = sc->fi_pos;
 803   if (NULL == res)
 804     {
 805       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 806                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 807                   p->filename,
 808                   _("failed to compute hash"));
 809       p->data.file.do_index = GNUNET_NO;
 810       GNUNET_FS_file_information_sync_ (p);
 811       publish_content (sc);
 812       return;
 813     }
 814   if (GNUNET_YES == p->data.file.index_start_confirmed)
 815     {
 816       publish_content (sc);
 817       return;
 818     }
 819   fn = GNUNET_STRINGS_filename_expand (p->filename);
 820   GNUNET_assert (fn != NULL);
 821   slen = strlen (fn) + 1;
 822   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 823     {
 824       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 825                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 826                   fn,
 827                   _("filename too long"));
 828       GNUNET_free (fn);
 829       p->data.file.do_index = GNUNET_NO;
 830       GNUNET_FS_file_information_sync_ (p);
 831       publish_content (sc);
 832       return;
 833     }
 834 #if DEBUG_PUBLISH
 835   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 836               "Hash of indexed file `%s' is `%s'\n",
 837               p->filename,
 838               GNUNET_h2s (res));
 839 #endif
 840   client = GNUNET_CLIENT_connect ("fs",
 841                                   sc->h->cfg);
 842   if (NULL == client)
 843     {
 844       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 845                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 846                   p->filename,
 847                   _("could not connect to `fs' service"));
 848       p->data.file.do_index = GNUNET_NO;
 849       publish_content (sc);
 850       GNUNET_free (fn);
 851       return;
 852     }
 853   if (p->data.file.have_hash != GNUNET_YES)
 854     {
 855       p->data.file.file_id = *res;
 856       p->data.file.have_hash = GNUNET_YES;
 857       GNUNET_FS_file_information_sync_ (p);
 858     }
 859   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 860                        slen);
 861   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 862                            slen);
 863   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 864   if (GNUNET_OK ==
 865       GNUNET_DISK_file_get_identifiers (p->filename,
 866                                         &dev,
 867                                         &ino))
 868     {
 869       ism->device = GNUNET_htonll (dev);
 870       ism->inode = GNUNET_htonll(ino);
 871     }
 872 #if DEBUG_PUBLISH
 873   else
 874     {
 875       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 876                   _("Failed to get file identifiers for `%s'\n"),
 877                   p->filename);
 878     }
 879 #endif
 880   ism->file_id = *res;
 881   memcpy (&ism[1],
 882           fn,
 883           slen);
 884   GNUNET_free (fn);
 885   sc->client = client;
 886   GNUNET_break (GNUNET_YES ==
 887                 GNUNET_CLIENT_transmit_and_get_response (client,
 888                                                          &ism->header,
 889                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 890                                                          GNUNET_YES,
 891                                                          &process_index_start_response,
 892                                                          sc));
 893   GNUNET_free (ism);
 894 }
 895
 896
 897 /**
 898  * Main function that performs the upload.
 899  *
 900  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 901  * @param tc task context
 902  */
 903 void
 904 GNUNET_FS_publish_main_ (void *cls,
 905                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 906 {
 907   struct GNUNET_FS_PublishContext *pc = cls;
 908   struct GNUNET_FS_ProgressInfo pi;
 909   struct GNUNET_FS_FileInformation *p;
 910   struct GNUNET_FS_Uri *loc;
 911   char *fn;
 912
 913   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 914   p = pc->fi_pos;
 915   if (NULL == p)
 916     {
 917 #if DEBUG_PUBLISH
 918       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 919                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 920 #endif
 921       /* upload of entire hierarchy complete,
 922          publish namespace entries */
 923       GNUNET_FS_publish_sync_ (pc);
 924       publish_sblock (pc);
 925       return;
 926     }
 927   /* find starting position */
 928   while ( (p->is_directory) &&
 929           (NULL != p->data.dir.entries) &&
 930           (NULL == p->emsg) &&
 931           (NULL == p->data.dir.entries->chk_uri) )
 932     {
 933       p = p->data.dir.entries;
 934       pc->fi_pos = p;
 935       GNUNET_FS_publish_sync_ (pc);
 936     }
 937   /* abort on error */
 938   if (NULL != p->emsg)
 939     {
 940 #if DEBUG_PUBLISH
 941       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 942                   "Error uploading: %s\n",
 943                   p->emsg);
 944 #endif
 945       /* error with current file, abort all
 946          related files as well! */
 947       while (NULL != p->dir)
 948         {
 949           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 950                                                        EXTRACTOR_METATYPE_FILENAME);
 951           p = p->dir;
 952           if (fn != NULL)
 953             {
 954               GNUNET_asprintf (&p->emsg,
 955                                _("Recursive upload failed at `%s': %s"),
 956                                fn,
 957                                p->emsg);
 958               GNUNET_free (fn);
 959             }
 960           else
 961             {
 962               GNUNET_asprintf (&p->emsg,
 963                                _("Recursive upload failed: %s"),
 964                                p->emsg);
 965             }
 966           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 967           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 968           pi.value.publish.specifics.error.message = p->emsg;
 969           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 970         }
 971       pc->all_done = GNUNET_YES;
 972       GNUNET_FS_publish_sync_ (pc);
 973       return;
 974     }
 975   /* handle completion */
 976   if (NULL != p->chk_uri)
 977     {
 978 #if DEBUG_PUBLISH
 979       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 980                   "File upload complete, now publishing KSK blocks.\n");
 981 #endif
 982       if (0 == p->anonymity)
 983         {
 984           /* zero anonymity, box CHK URI in LOC URI */
 985           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
 986                                           pc->h->cfg,
 987                                           p->expirationTime);
 988           GNUNET_FS_uri_destroy (p->chk_uri);
 989           p->chk_uri = loc;
 990         }
 991       GNUNET_FS_publish_sync_ (pc);
 992       /* upload of "p" complete, publish KBlocks! */
 993       if (p->keywords != NULL)
 994         {
 995           GNUNET_FS_publish_ksk (pc->h,
 996                                  p->keywords,
 997                                  p->meta,
 998                                  p->chk_uri,
 999                                  p->expirationTime,
1000                                  p->anonymity,
1001                                  p->priority,
1002                                  pc->options,
1003                                  &publish_kblocks_cont,
1004                                  pc);
1005         }
1006       else
1007         {
1008           publish_kblocks_cont (pc,
1009                                 p->chk_uri,
1010                                 NULL);
1011         }
1012       return;
1013     }
1014   if ( (!p->is_directory) &&
1015        (p->data.file.do_index) )
1016     {
1017       if (NULL == p->filename)
1018         {
1019           p->data.file.do_index = GNUNET_NO;
1020           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1021                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1022                       "<no-name>",
1023                       _("needs to be an actual file"));
1024           GNUNET_FS_file_information_sync_ (p);
1025           publish_content (pc);
1026           return;
1027         }
1028       if (p->data.file.have_hash)
1029         {
1030           hash_for_index_cb (pc,
1031                              &p->data.file.file_id);
1032         }
1033       else
1034         {
1035           p->start_time = GNUNET_TIME_absolute_get ();
1036           pc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
1037                                              p->filename,
1038                                              HASHING_BLOCKSIZE,
1039                                              &hash_for_index_cb,
1040                                              pc);
1041         }
1042       return;
1043     }
1044   publish_content (pc);
1045 }
1046
1047
1048 /**
1049  * Signal the FS's progress function that we are starting
1050  * an upload.
1051  *
1052  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1053  * @param fi the entry in the publish-structure
1054  * @param length length of the file or directory
1055  * @param meta metadata for the file or directory (can be modified)
1056  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1057  * @param anonymity pointer to selected anonymity level (can be modified)
1058  * @param priority pointer to selected priority (can be modified)
1059  * @param do_index should we index?
1060  * @param expirationTime pointer to selected expiration time (can be modified)
1061  * @param client_info pointer to client context set upon creation (can be modified)
1062  * @return GNUNET_OK to continue (always)
1063  */
1064 static int
1065 fip_signal_start(void *cls,
1066                  struct GNUNET_FS_FileInformation *fi,
1067                  uint64_t length,
1068                  struct GNUNET_CONTAINER_MetaData *meta,
1069                  struct GNUNET_FS_Uri **uri,
1070                  uint32_t *anonymity,
1071                  uint32_t *priority,
1072                  int *do_index,
1073                  struct GNUNET_TIME_Absolute *expirationTime,
1074                  void **client_info)
1075 {
1076   struct GNUNET_FS_PublishContext *sc = cls;
1077   struct GNUNET_FS_ProgressInfo pi;
1078   unsigned int kc;
1079   uint64_t left;
1080
1081   if (*do_index)
1082     {
1083       /* space for on-demand blocks */
1084       sc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1085     }
1086   else
1087     {
1088       /* space for DBlocks */
1089       sc->reserve_space += length;
1090     }
1091   /* entries for IBlocks and DBlocks, space for IBlocks */
1092   left = length;
1093   while (1)
1094     {
1095       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1096       sc->reserve_entries += left;
1097       if (left <= 1)
1098         break;
1099       left = left * sizeof (struct ContentHashKey);
1100       sc->reserve_space += left;
1101     }
1102   sc->reserve_entries++;
1103   /* entries and space for keywords */
1104   if (NULL != *uri)
1105     {
1106       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1107       sc->reserve_entries += kc;
1108       sc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1109     }
1110   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1111   *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
1112   GNUNET_FS_file_information_sync_ (fi);
1113   return GNUNET_OK;
1114 }
1115
1116
1117 /**
1118  * Signal the FS's progress function that we are suspending
1119  * an upload.
1120  *
1121  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1122  * @param fi the entry in the publish-structure
1123  * @param length length of the file or directory
1124  * @param meta metadata for the file or directory (can be modified)
1125  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1126  * @param anonymity pointer to selected anonymity level (can be modified)
1127  * @param priority pointer to selected priority (can be modified)
1128  * @param do_index should we index?
1129  * @param expirationTime pointer to selected expiration time (can be modified)
1130  * @param client_info pointer to client context set upon creation (can be modified)
1131  * @return GNUNET_OK to continue (always)
1132  */
1133 static int
1134 fip_signal_suspend(void *cls,
1135                    struct GNUNET_FS_FileInformation *fi,
1136                    uint64_t length,
1137                    struct GNUNET_CONTAINER_MetaData *meta,
1138                    struct GNUNET_FS_Uri **uri,
1139                    uint32_t *anonymity,
1140                    uint32_t *priority,
1141                    int *do_index,
1142                    struct GNUNET_TIME_Absolute *expirationTime,
1143                    void **client_info)
1144 {
1145   struct GNUNET_FS_PublishContext*sc = cls;
1146   struct GNUNET_FS_ProgressInfo pi;
1147   uint64_t off;
1148
1149   GNUNET_free_non_null (fi->serialization);
1150   fi->serialization = NULL;
1151   off = (fi->chk_uri == NULL) ? 0 : length;
1152   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1153   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1154   *client_info = NULL;
1155   if (NULL != sc->dsh)
1156     {
1157       GNUNET_DATASTORE_disconnect (sc->dsh, GNUNET_NO);
1158       sc->dsh = NULL;
1159     }
1160   return GNUNET_OK;
1161 }
1162
1163
1164 /**
1165  * Create SUSPEND event for the given publish operation
1166  * and then clean up our state (without stop signal).
1167  *
1168  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1169  */
1170 void
1171 GNUNET_FS_publish_signal_suspend_ (void *cls)
1172 {
1173   struct GNUNET_FS_PublishContext *pc = cls;
1174
1175   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1176     {
1177       GNUNET_SCHEDULER_cancel (pc->upload_task);
1178       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1179     }
1180   GNUNET_FS_file_information_inspect (pc->fi,
1181                                       &fip_signal_suspend,
1182                                       pc);
1183   GNUNET_FS_end_top (pc->h, pc->top);
1184   publish_cleanup (pc, NULL);
1185 }
1186
1187
1188 /**
1189  * We have gotten a reply for our space reservation request.
1190  * Either fail (insufficient space) or start publishing for good.
1191  *
1192  * @param cls the 'struct GNUNET_FS_PublishContext*'
1193  * @param success positive reservation ID on success
1194  * @param msg error message on error, otherwise NULL
1195  */
1196 static void
1197 finish_reserve (void *cls,
1198                 int success,
1199                 const char *msg)
1200 {
1201   struct GNUNET_FS_PublishContext *pc = cls;
1202
1203   pc->qre = NULL;
1204   if ( (msg != NULL) ||
1205        (success <= 0) )
1206     {
1207       GNUNET_asprintf (&pc->fi->emsg,
1208                        _("Insufficient space for publishing: %s"),
1209                        msg);
1210       signal_publish_error (pc->fi,
1211                             pc,
1212                             pc->fi->emsg);
1213       return;
1214     }
1215   pc->rid = success;
1216   pc->upload_task
1217     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1218                                           &GNUNET_FS_publish_main_,
1219                                           pc);
1220 }
1221
1222
1223 /**
1224  * Publish a file or directory.
1225  *
1226  * @param h handle to the file sharing subsystem
1227  * @param fi information about the file or directory structure to publish
1228  * @param namespace namespace to publish the file in, NULL for no namespace
1229  * @param nid identifier to use for the publishd content in the namespace
1230  *        (can be NULL, must be NULL if namespace is NULL)
1231  * @param nuid update-identifier that will be used for future updates
1232  *        (can be NULL, must be NULL if namespace or nid is NULL)
1233  * @param options options for the publication
1234  * @return context that can be used to control the publish operation
1235  */
1236 struct GNUNET_FS_PublishContext *
1237 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1238                          struct GNUNET_FS_FileInformation *fi,
1239                          struct GNUNET_FS_Namespace *namespace,
1240                          const char *nid,
1241                          const char *nuid,
1242                          enum GNUNET_FS_PublishOptions options)
1243 {
1244   struct GNUNET_FS_PublishContext *ret;
1245   struct GNUNET_DATASTORE_Handle *dsh;
1246
1247   GNUNET_assert (NULL != h);
1248   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1249     {
1250       dsh = GNUNET_DATASTORE_connect (h->cfg);
1251       if (NULL == dsh)
1252         return NULL;
1253     }
1254   else
1255     {
1256       dsh = NULL;
1257     }
1258   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1259   ret->dsh = dsh;
1260   ret->h = h;
1261   ret->fi = fi;
1262   ret->namespace = namespace;
1263   if (namespace != NULL)
1264     {
1265       namespace->rc++;
1266       GNUNET_assert (NULL != nid);
1267       ret->nid = GNUNET_strdup (nid);
1268       if (NULL != nuid)
1269         ret->nuid = GNUNET_strdup (nuid);
1270     }
1271   /* signal start */
1272   GNUNET_FS_file_information_inspect (ret->fi,
1273                                       &fip_signal_start,
1274                                       ret);
1275   ret->fi_pos = ret->fi;
1276   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1277   GNUNET_FS_publish_sync_ (ret);
1278   if (NULL != ret->dsh)
1279     {
1280       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1281                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1282                   (unsigned int) ret->reserve_entries,
1283                   (unsigned long long) ret->reserve_space);
1284       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1285                                            ret->reserve_space,
1286                                            ret->reserve_entries,
1287                                            UINT_MAX,
1288                                            UINT_MAX,
1289                                            GNUNET_TIME_UNIT_FOREVER_REL,
1290                                            &finish_reserve,
1291                                            ret);
1292     }
1293   else
1294     {
1295       ret->upload_task
1296         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1297                                               &GNUNET_FS_publish_main_,
1298                                               ret);
1299     }
1300   return ret;
1301 }
1302
1303
1304 /**
1305  * Signal the FS's progress function that we are stopping
1306  * an upload.
1307  *
1308  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1309  * @param fi the entry in the publish-structure
1310  * @param length length of the file or directory
1311  * @param meta metadata for the file or directory (can be modified)
1312  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1313  * @param anonymity pointer to selected anonymity level (can be modified)
1314  * @param priority pointer to selected priority (can be modified)
1315  * @param do_index should we index?
1316  * @param expirationTime pointer to selected expiration time (can be modified)
1317  * @param client_info pointer to client context set upon creation (can be modified)
1318  * @return GNUNET_OK to continue (always)
1319  */
1320 static int
1321 fip_signal_stop(void *cls,
1322                 struct GNUNET_FS_FileInformation *fi,
1323                 uint64_t length,
1324                 struct GNUNET_CONTAINER_MetaData *meta,
1325                 struct GNUNET_FS_Uri **uri,
1326                 uint32_t *anonymity,
1327                 uint32_t *priority,
1328                 int *do_index,
1329                 struct GNUNET_TIME_Absolute *expirationTime,
1330                 void **client_info)
1331 {
1332   struct GNUNET_FS_PublishContext*sc = cls;
1333   struct GNUNET_FS_ProgressInfo pi;
1334   uint64_t off;
1335
1336   if (fi->serialization != NULL)
1337     {
1338       GNUNET_FS_remove_sync_file_ (sc->h,
1339                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1340                                    fi->serialization);
1341       GNUNET_free (fi->serialization);
1342       fi->serialization = NULL;
1343     }
1344   off = (fi->chk_uri == NULL) ? 0 : length;
1345   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1346   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1347   *client_info = NULL;
1348   return GNUNET_OK;
1349 }
1350
1351
1352 /**
1353  * Stop an upload.  Will abort incomplete uploads (but
1354  * not remove blocks that have already been publishd) or
1355  * simply clean up the state for completed uploads.
1356  * Must NOT be called from within the event callback!
1357  *
1358  * @param pc context for the upload to stop
1359  */
1360 void
1361 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1362 {
1363   GNUNET_FS_end_top (pc->h, pc->top);
1364   if (NULL != pc->qre)
1365     {
1366       GNUNET_DATASTORE_cancel (pc->qre);
1367       pc->qre = NULL;
1368     }
1369   if (NULL != pc->dsh)
1370     {
1371       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1372       pc->dsh = NULL;
1373     }
1374   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1375     {
1376       GNUNET_SCHEDULER_cancel (pc->upload_task);
1377       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1378     }
1379   if (pc->serialization != NULL)
1380     {
1381       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1382       GNUNET_free (pc->serialization);
1383       pc->serialization = NULL;
1384     }
1385   GNUNET_FS_file_information_inspect (pc->fi,
1386                                       &fip_signal_stop,
1387                                       pc);
1388   if (GNUNET_YES == pc->in_network_wait)
1389     {
1390       pc->in_network_wait = GNUNET_SYSERR;
1391       return;
1392     }
1393   publish_cleanup (pc, NULL);
1394 }
1395
1396
1397 /**
1398  * Context for the KSK publication.
1399  */
1400 struct PublishKskContext
1401 {
1402
1403   /**
1404    * Keywords to use.
1405    */
1406   struct GNUNET_FS_Uri *ksk_uri;
1407
1408   /**
1409    * Global FS context.
1410    */
1411   struct GNUNET_FS_Handle *h;
1412
1413   /**
1414    * The master block that we are sending
1415    * (in plaintext), has "mdsize+slen" more
1416    * bytes than the struct would suggest.
1417    */
1418   struct KBlock *kb;
1419
1420   /**
1421    * Buffer of the same size as "kb" for
1422    * the encrypted version.
1423    */
1424   struct KBlock *cpy;
1425
1426   /**
1427    * Handle to the datastore, NULL if we are just
1428    * simulating.
1429    */
1430   struct GNUNET_DATASTORE_Handle *dsh;
1431
1432   /**
1433    * Function to call once we're done.
1434    */
1435   GNUNET_FS_PublishContinuation cont;
1436
1437   /**
1438    * Closure for cont.
1439    */
1440   void *cont_cls;
1441
1442   /**
1443    * When should the KBlocks expire?
1444    */
1445   struct GNUNET_TIME_Absolute expirationTime;
1446
1447   /**
1448    * Size of the serialized metadata.
1449    */
1450   ssize_t mdsize;
1451
1452   /**
1453    * Size of the (CHK) URI as a string.
1454    */
1455   size_t slen;
1456
1457   /**
1458    * Keyword that we are currently processing.
1459    */
1460   unsigned int i;
1461
1462   /**
1463    * Anonymity level for the KBlocks.
1464    */
1465   uint32_t anonymity;
1466
1467   /**
1468    * Priority for the KBlocks.
1469    */
1470   uint32_t priority;
1471 };
1472
1473
1474 /**
1475  * Continuation of "GNUNET_FS_publish_ksk" that performs
1476  * the actual publishing operation (iterating over all
1477  * of the keywords).
1478  *
1479  * @param cls closure of type "struct PublishKskContext*"
1480  * @param tc unused
1481  */
1482 static void
1483 publish_ksk_cont (void *cls,
1484                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1485
1486
1487 /**
1488  * Function called by the datastore API with
1489  * the result from the PUT request.
1490  *
1491  * @param cls closure of type "struct PublishKskContext*"
1492  * @param success GNUNET_OK on success
1493  * @param msg error message (or NULL)
1494  */
1495 static void
1496 kb_put_cont (void *cls,
1497              int success,
1498              const char *msg)
1499 {
1500   struct PublishKskContext *pkc = cls;
1501
1502   if (GNUNET_OK != success)
1503     {
1504       if (NULL != pkc->dsh)
1505         {
1506           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1507           pkc->dsh = NULL;
1508         }
1509       GNUNET_free (pkc->cpy);
1510       GNUNET_free (pkc->kb);
1511       pkc->cont (pkc->cont_cls,
1512                  NULL,
1513                  msg);
1514       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1515       GNUNET_free (pkc);
1516       return;
1517     }
1518   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1519                                      pkc,
1520                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1521 }
1522
1523
1524 /**
1525  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1526  * publishing operation (iterating over all of the keywords).
1527  *
1528  * @param cls closure of type "struct PublishKskContext*"
1529  * @param tc unused
1530  */
1531 static void
1532 publish_ksk_cont (void *cls,
1533                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1534 {
1535   struct PublishKskContext *pkc = cls;
1536   const char *keyword;
1537   GNUNET_HashCode key;
1538   GNUNET_HashCode query;
1539   struct GNUNET_CRYPTO_AesSessionKey skey;
1540   struct GNUNET_CRYPTO_AesInitializationVector iv;
1541   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1542
1543
1544   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1545        (NULL == pkc->dsh) )
1546     {
1547       if (NULL != pkc->dsh)
1548         {
1549           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1550           pkc->dsh = NULL;
1551         }
1552       GNUNET_free (pkc->cpy);
1553       GNUNET_free (pkc->kb);
1554       pkc->cont (pkc->cont_cls,
1555                  pkc->ksk_uri,
1556                  NULL);
1557       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1558       GNUNET_free (pkc);
1559       return;
1560     }
1561   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1562   /* first character of keyword indicates if it is
1563      mandatory or not -- ignore for hashing */
1564   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1565   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1566   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1567                              pkc->slen + pkc->mdsize,
1568                              &skey,
1569                              &iv,
1570                              &pkc->cpy[1]);
1571   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1572   GNUNET_assert (NULL != pk);
1573   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1574   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1575                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1576                       &query);
1577   GNUNET_assert (GNUNET_OK ==
1578                  GNUNET_CRYPTO_rsa_sign (pk,
1579                                          &pkc->cpy->purpose,
1580                                          &pkc->cpy->signature));
1581   GNUNET_CRYPTO_rsa_key_free (pk);
1582   GNUNET_DATASTORE_put (pkc->dsh,
1583                         0,
1584                         &query,
1585                         pkc->mdsize +
1586                         sizeof (struct KBlock) +
1587                         pkc->slen,
1588                         pkc->cpy,
1589                         GNUNET_BLOCK_TYPE_FS_KBLOCK,
1590                         pkc->priority,
1591                         pkc->anonymity,
1592                         pkc->expirationTime,
1593                         -2, 1,
1594                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1595                         &kb_put_cont,
1596                         pkc);
1597 }
1598
1599
1600 /**
1601  * Publish a CHK under various keywords on GNUnet.
1602  *
1603  * @param h handle to the file sharing subsystem
1604  * @param ksk_uri keywords to use
1605  * @param meta metadata to use
1606  * @param uri URI to refer to in the KBlock
1607  * @param expirationTime when the KBlock expires
1608  * @param anonymity anonymity level for the KBlock
1609  * @param priority priority for the KBlock
1610  * @param options publication options
1611  * @param cont continuation
1612  * @param cont_cls closure for cont
1613  */
1614 void
1615 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1616                        const struct GNUNET_FS_Uri *ksk_uri,
1617                        const struct GNUNET_CONTAINER_MetaData *meta,
1618                        const struct GNUNET_FS_Uri *uri,
1619                        struct GNUNET_TIME_Absolute expirationTime,
1620                        uint32_t anonymity,
1621                        uint32_t priority,
1622                        enum GNUNET_FS_PublishOptions options,
1623                        GNUNET_FS_PublishContinuation cont,
1624                        void *cont_cls)
1625 {
1626   struct PublishKskContext *pkc;
1627   char *uris;
1628   size_t size;
1629   char *kbe;
1630   char *sptr;
1631
1632   GNUNET_assert (NULL != uri);
1633   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1634   pkc->h = h;
1635   pkc->expirationTime = expirationTime;
1636   pkc->anonymity = anonymity;
1637   pkc->priority = priority;
1638   pkc->cont = cont;
1639   pkc->cont_cls = cont_cls;
1640   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1641     {
1642       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
1643       if (pkc->dsh == NULL)
1644         {
1645           cont (cont_cls, NULL, _("Could not connect to datastore."));
1646           GNUNET_free (pkc);
1647           return;
1648         }
1649     }
1650   if (meta == NULL)
1651     pkc->mdsize = 0;
1652   else
1653     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1654   GNUNET_assert (pkc->mdsize >= 0);
1655   uris = GNUNET_FS_uri_to_string (uri);
1656   pkc->slen = strlen (uris) + 1;
1657   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1658   if (size > MAX_KBLOCK_SIZE)
1659     {
1660       size = MAX_KBLOCK_SIZE;
1661       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1662     }
1663   pkc->kb = GNUNET_malloc (size);
1664   kbe = (char *) &pkc->kb[1];
1665   memcpy (kbe, uris, pkc->slen);
1666   GNUNET_free (uris);
1667   sptr = &kbe[pkc->slen];
1668   if (meta != NULL)
1669     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1670                                                         &sptr,
1671                                                         pkc->mdsize,
1672                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1673   if (pkc->mdsize == -1)
1674     {
1675       GNUNET_break (0);
1676       GNUNET_free (pkc->kb);
1677       if (pkc->dsh != NULL)
1678         {
1679           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1680           pkc->dsh = NULL;
1681         }
1682       cont (cont_cls, NULL, _("Internal error."));
1683       GNUNET_free (pkc);
1684       return;
1685     }
1686   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1687
1688   pkc->cpy = GNUNET_malloc (size);
1689   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1690                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1691                                   pkc->mdsize +
1692                                   pkc->slen);
1693   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1694   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1695   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1696                                      pkc,
1697                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1698 }
1699
1700
1701 /* end of fs_publish.c */