src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see http://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  *
  28  * TODO:
  29  * - indexing cleanup: unindex on failure (can wait)
  30  * - datastore reservation support (optimization)
  31  * - location URIs (publish with anonymity-level zero)
  32  */
  33
  34 #include "platform.h"
  35 #include "gnunet_constants.h"
  36 #include "gnunet_signatures.h"
  37 #include "gnunet_util_lib.h"
  38 #include "gnunet_fs_service.h"
  39 #include "fs.h"
  40 #include "fs_tree.h"
  41
  42 #define DEBUG_PUBLISH GNUNET_NO
  43
  44
  45 /**
  46  * Context for "ds_put_cont".
  47  */
  48 struct PutContCtx
  49 {
  50   /**
  51    * Current publishing context.
  52    */
  53   struct GNUNET_FS_PublishContext *sc;
  54
  55   /**
  56    * Specific file with the block.
  57    */
  58   struct GNUNET_FS_FileInformation *p;
  59
  60   /**
  61    * Function to run next, if any (can be NULL).
  62    */
  63   GNUNET_SCHEDULER_Task cont;
  64
  65   /**
  66    * Closure for cont.
  67    */
  68   void *cont_cls;
  69 };
  70
  71
  72 /**
  73  * Fill in all of the generic fields for
  74  * a publish event and call the callback.
  75  *
  76  * @param pi structure to fill in
  77  * @param sc overall publishing context
  78  * @param p file information for the file being published
  79  * @param offset where in the file are we so far
  80  * @return value returned from callback
  81  */
  82 void *
  83 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  84                                 struct GNUNET_FS_PublishContext *sc,
  85                                 const struct GNUNET_FS_FileInformation *p,
  86                                 uint64_t offset)
  87 {
  88   pi->value.publish.pc = sc;
  89   pi->value.publish.fi = p;
  90   pi->value.publish.cctx
  91     = p->client_info;
  92   pi->value.publish.pctx
  93     = (NULL == p->dir) ? NULL : p->dir->client_info;
  94   pi->value.publish.filename = p->filename;
  95   pi->value.publish.size
  96     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  97   pi->value.publish.eta
  98     = GNUNET_TIME_calculate_eta (p->start_time,
  99                                  offset,
 100                                  pi->value.publish.size);
 101   pi->value.publish.completed = offset;
 102   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
 103   pi->value.publish.anonymity = p->anonymity;
 104   return sc->h->upcb (sc->h->upcb_cls,
 105                       pi);
 106 }
 107
 108
 109 /**
 110  * Cleanup the publish context, we're done with it.
 111  *
 112  * @param cls struct to clean up after
 113  * @param tc scheduler context
 114  */
 115 static void
 116 publish_cleanup (void *cls,
 117                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 118 {
 119   struct GNUNET_FS_PublishContext *pc = cls;
 120
 121   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 122   if (pc->namespace != NULL)
 123     GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 124   GNUNET_free_non_null (pc->nid);
 125   GNUNET_free_non_null (pc->nuid);
 126   GNUNET_free_non_null (pc->serialization);
 127   if (pc->dsh != NULL)
 128     {
 129       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 130       pc->dsh = NULL;
 131     }
 132   if (pc->client != NULL)
 133     GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 134   GNUNET_free (pc);
 135 }
 136
 137
 138 /**
 139  * Function called by the datastore API with
 140  * the result from the PUT request.
 141  *
 142  * @param cls our closure
 143  * @param success GNUNET_OK on success
 144  * @param msg error message (or NULL)
 145  */
 146 static void
 147 ds_put_cont (void *cls,
 148              int success,
 149              const char *msg)
 150 {
 151   struct PutContCtx *pcc = cls;
 152   struct GNUNET_FS_ProgressInfo pi;
 153
 154   if (GNUNET_SYSERR == pcc->sc->in_network_wait)
 155     {
 156       /* we were aborted in the meantime, finish shutdown! */
 157       GNUNET_SCHEDULER_add_continuation (pcc->sc->h->sched,
 158                                          &publish_cleanup,
 159                                          pcc->sc,
 160                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 161       GNUNET_free (pcc);
 162       return;
 163     }
 164   GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait);
 165   pcc->sc->in_network_wait = GNUNET_NO;
 166   if (GNUNET_OK != success)
 167     {
 168       GNUNET_asprintf (&pcc->p->emsg,
 169                        _("Publishing failed: %s"),
 170                        msg);
 171       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 172       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 173       pi.value.publish.specifics.error.message = pcc->p->emsg;
 174       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0);
 175     }
 176   if (NULL != pcc->cont)
 177     pcc->sc->upload_task
 178       = GNUNET_SCHEDULER_add_with_priority (pcc->sc->h->sched,
 179                                             GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 180                                             pcc->cont,
 181                                             pcc->cont_cls);
 182   GNUNET_free (pcc);
 183 }
 184
 185
 186 /**
 187  * Generate the callback that signals clients
 188  * that a file (or directory) has been completely
 189  * published.
 190  *
 191  * @param p the completed upload
 192  * @param sc context of the publication
 193  */
 194 static void
 195 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 196                            struct GNUNET_FS_PublishContext *sc)
 197 {
 198   struct GNUNET_FS_ProgressInfo pi;
 199
 200   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 201   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 202   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 203   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 204                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 205 }
 206
 207
 208 /**
 209  * Generate the callback that signals clients
 210  * that a file (or directory) has encountered
 211  * a problem during publication.
 212  *
 213  * @param p the upload that had trouble
 214  * @param sc context of the publication
 215  * @param emsg error message
 216  */
 217 static void
 218 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 219                       struct GNUNET_FS_PublishContext *sc,
 220                       const char *emsg)
 221 {
 222   struct GNUNET_FS_ProgressInfo pi;
 223
 224   p->emsg = GNUNET_strdup (emsg);
 225   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 226   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 227   pi.value.publish.specifics.error.message =emsg;
 228   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 229 }
 230
 231
 232 /**
 233  * Datastore returns from reservation cancel request.
 234  *
 235  * @param cls the 'struct GNUNET_FS_PublishContext'
 236  * @param success success code (not used)
 237  * @param msg error message (typically NULL, not used)
 238  */
 239 static void
 240 finish_release_reserve (void *cls,
 241                         int success,
 242                         const char *msg)
 243 {
 244   struct GNUNET_FS_PublishContext *pc = cls;
 245
 246   pc->qre = NULL;
 247   signal_publish_completion (pc->fi, pc);
 248   pc->all_done = GNUNET_YES;
 249   GNUNET_FS_publish_sync_ (pc);
 250 }
 251
 252
 253 /**
 254  * We've finished publishing the SBlock as part of a larger upload.
 255  * Check the result and complete the larger upload.
 256  *
 257  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 258  * @param uri URI of the published SBlock
 259  * @param emsg NULL on success, otherwise error message
 260  */
 261 static void
 262 publish_sblocks_cont (void *cls,
 263                       const struct GNUNET_FS_Uri *uri,
 264                       const char *emsg)
 265 {
 266   struct GNUNET_FS_PublishContext *pc = cls;
 267   if (NULL != emsg)
 268     {
 269       signal_publish_error (pc->fi,
 270                             pc,
 271                             emsg);
 272       GNUNET_FS_publish_sync_ (pc);
 273       return;
 274     }
 275   GNUNET_assert (pc->qre == NULL);
 276   if ( (pc->dsh != NULL) &&
 277        (pc->rid != 0) )
 278     {
 279       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 280                                                   pc->rid,
 281                                                   UINT_MAX,
 282                                                   UINT_MAX,
 283                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 284                                                   &finish_release_reserve,
 285                                                   pc);
 286     }
 287   else
 288     {
 289       finish_release_reserve (pc, GNUNET_OK, NULL);
 290     }
 291 }
 292
 293
 294 /**
 295  * We are almost done publishing the structure,
 296  * add SBlocks (if needed).
 297  *
 298  * @param sc overall upload data
 299  */
 300 static void
 301 publish_sblock (struct GNUNET_FS_PublishContext *sc)
 302 {
 303   if (NULL != sc->namespace)
 304     GNUNET_FS_publish_sks (sc->h,
 305                            sc->namespace,
 306                            sc->nid,
 307                            sc->nuid,
 308                            sc->fi->meta,
 309                            sc->fi->chk_uri,
 310                            sc->fi->expirationTime,
 311                            sc->fi->anonymity,
 312                            sc->fi->priority,
 313                            sc->options,
 314                            &publish_sblocks_cont,
 315                            sc);
 316   else
 317     publish_sblocks_cont (sc, NULL, NULL);
 318 }
 319
 320
 321 /**
 322  * We've finished publishing a KBlock as part of a larger upload.
 323  * Check the result and continue the larger upload.
 324  *
 325  * @param cls the "struct GNUNET_FS_PublishContext*"
 326  *        of the larger upload
 327  * @param uri URI of the published blocks
 328  * @param emsg NULL on success, otherwise error message
 329  */
 330 static void
 331 publish_kblocks_cont (void *cls,
 332                       const struct GNUNET_FS_Uri *uri,
 333                       const char *emsg)
 334 {
 335   struct GNUNET_FS_PublishContext *pc = cls;
 336   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 337
 338   if (NULL != emsg)
 339     {
 340 #if DEBUG_PUBLISH
 341       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 342                   "Error uploading KSK blocks: %s\n",
 343                   emsg);
 344 #endif
 345       signal_publish_error (p, pc, emsg);
 346       GNUNET_FS_file_information_sync_ (p);
 347       GNUNET_FS_publish_sync_ (pc);
 348       pc->upload_task
 349         = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 350                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 351                                               &GNUNET_FS_publish_main_,
 352                                               pc);
 353       return;
 354     }
 355 #if DEBUG_PUBLISH
 356   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 357               "KSK blocks published, moving on to next file\n");
 358 #endif
 359   if (NULL != p->dir)
 360     signal_publish_completion (p, pc);
 361   /* move on to next file */
 362   if (NULL != p->next)
 363     pc->fi_pos = p->next;
 364   else
 365     pc->fi_pos = p->dir;
 366   GNUNET_FS_publish_sync_ (pc);
 367   pc->upload_task
 368     = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 369                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 370                                           &GNUNET_FS_publish_main_,
 371                                           pc);
 372 }
 373
 374
 375 /**
 376  * Function called by the tree encoder to obtain
 377  * a block of plaintext data (for the lowest level
 378  * of the tree).
 379  *
 380  * @param cls our publishing context
 381  * @param offset identifies which block to get
 382  * @param max (maximum) number of bytes to get; returning
 383  *        fewer will also cause errors
 384  * @param buf where to copy the plaintext buffer
 385  * @param emsg location to store an error message (on error)
 386  * @return number of bytes copied to buf, 0 on error
 387  */
 388 static size_t
 389 block_reader (void *cls,
 390               uint64_t offset,
 391               size_t max,
 392               void *buf,
 393               char **emsg)
 394 {
 395   struct GNUNET_FS_PublishContext *sc = cls;
 396   struct GNUNET_FS_FileInformation *p;
 397   size_t pt_size;
 398   const char *dd;
 399
 400   p = sc->fi_pos;
 401   if (p->is_directory)
 402     {
 403       pt_size = GNUNET_MIN(max,
 404                            p->data.dir.dir_size - offset);
 405       dd = p->data.dir.dir_data;
 406       memcpy (buf,
 407               &dd[offset],
 408               pt_size);
 409     }
 410   else
 411     {
 412       pt_size = GNUNET_MIN(max,
 413                            p->data.file.file_size - offset);
 414       if (pt_size == 0)
 415         return 0; /* calling reader with pt_size==0
 416                      might free buf, so don't! */
 417       if (pt_size !=
 418           p->data.file.reader (p->data.file.reader_cls,
 419                                offset,
 420                                pt_size,
 421                                buf,
 422                                emsg))
 423         return 0;
 424     }
 425   return pt_size;
 426 }
 427
 428
 429 /**
 430  * The tree encoder has finished processing a
 431  * file.   Call it's finish method and deal with
 432  * the final result.
 433  *
 434  * @param cls our publishing context
 435  * @param tc scheduler's task context (not used)
 436  */
 437 static void
 438 encode_cont (void *cls,
 439              const struct GNUNET_SCHEDULER_TaskContext *tc)
 440 {
 441   struct GNUNET_FS_PublishContext *sc = cls;
 442   struct GNUNET_FS_FileInformation *p;
 443   struct GNUNET_FS_ProgressInfo pi;
 444   char *emsg;
 445
 446   p = sc->fi_pos;
 447   GNUNET_FS_tree_encoder_finish (p->te,
 448                                  &p->chk_uri,
 449                                  &emsg);
 450   p->te = NULL;
 451   if (NULL != emsg)
 452     {
 453 #if DEBUG_PUBLISH
 454       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 455                   "Error during tree walk: %s\n",
 456                   emsg);
 457 #endif
 458       GNUNET_asprintf (&p->emsg,
 459                        _("Publishing failed: %s"),
 460                        emsg);
 461       GNUNET_free (emsg);
 462       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 463       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 464       pi.value.publish.specifics.error.message = p->emsg;
 465       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 466     }
 467 #if DEBUG_PUBLISH
 468   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 469               "Finished with tree encoder\n");
 470 #endif
 471   /* continue with main */
 472   sc->upload_task
 473     = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 474                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 475                                           &GNUNET_FS_publish_main_,
 476                                           sc);
 477 }
 478
 479
 480 /**
 481  * Function called asking for the current (encoded)
 482  * block to be processed.  After processing the
 483  * client should either call "GNUNET_FS_tree_encode_next"
 484  * or (on error) "GNUNET_FS_tree_encode_finish".
 485  *
 486  * @param cls closure
 487  * @param query the query for the block (key for lookup in the datastore)
 488  * @param offset offset of the block in the file
 489  * @param type type of the block (IBLOCK or DBLOCK)
 490  * @param block the (encrypted) block
 491  * @param block_size size of block (in bytes)
 492  */
 493 static void
 494 block_proc (void *cls,
 495             const GNUNET_HashCode *query,
 496             uint64_t offset,
 497             enum GNUNET_BLOCK_Type type,
 498             const void *block,
 499             uint16_t block_size)
 500 {
 501   struct GNUNET_FS_PublishContext *sc = cls;
 502   struct GNUNET_FS_FileInformation *p;
 503   struct PutContCtx * dpc_cls;
 504   struct OnDemandBlock odb;
 505
 506   p = sc->fi_pos;
 507   if (NULL == sc->dsh)
 508     {
 509 #if DEBUG_PUBLISH
 510       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 511                   "Waiting for datastore connection\n");
 512 #endif
 513       sc->upload_task
 514         = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 515                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 516                                               &GNUNET_FS_publish_main_,
 517                                               sc);
 518       return;
 519     }
 520
 521   GNUNET_assert (GNUNET_NO == sc->in_network_wait);
 522   sc->in_network_wait = GNUNET_YES;
 523   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 524   dpc_cls->cont = &GNUNET_FS_publish_main_;
 525   dpc_cls->cont_cls = sc;
 526   dpc_cls->sc = sc;
 527   dpc_cls->p = p;
 528   if ( (! p->is_directory) &&
 529        (GNUNET_YES == p->data.file.do_index) &&
 530        (type == GNUNET_BLOCK_TYPE_DBLOCK) )
 531     {
 532 #if DEBUG_PUBLISH
 533       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 534                   "Indexing block `%s' for offset %llu with index size %u\n",
 535                   GNUNET_h2s (query),
 536                   (unsigned long long) offset,
 537                   sizeof (struct OnDemandBlock));
 538 #endif
 539       odb.offset = GNUNET_htonll (offset);
 540       odb.file_id = p->data.file.file_id;
 541       GNUNET_DATASTORE_put (sc->dsh,
 542                             sc->rid,
 543                             query,
 544                             sizeof(struct OnDemandBlock),
 545                             &odb,
 546                             GNUNET_BLOCK_TYPE_ONDEMAND,
 547                             p->priority,
 548                             p->anonymity,
 549                             p->expirationTime,
 550                             -2, 1,
 551                             GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 552                             &ds_put_cont,
 553                             dpc_cls);
 554       return;
 555     }
 556 #if DEBUG_PUBLISH
 557   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 558               "Publishing block `%s' for offset %llu with size %u\n",
 559               GNUNET_h2s (query),
 560               (unsigned long long) offset,
 561               (unsigned int) block_size);
 562 #endif
 563   GNUNET_DATASTORE_put (sc->dsh,
 564                         sc->rid,
 565                         query,
 566                         block_size,
 567                         block,
 568                         type,
 569                         p->priority,
 570                         p->anonymity,
 571                         p->expirationTime,
 572                         -2, 1,
 573                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 574                         &ds_put_cont,
 575                         dpc_cls);
 576 }
 577
 578
 579 /**
 580  * Function called with information about our
 581  * progress in computing the tree encoding.
 582  *
 583  * @param cls closure
 584  * @param offset where are we in the file
 585  * @param pt_block plaintext of the currently processed block
 586  * @param pt_size size of pt_block
 587  * @param depth depth of the block in the tree
 588  */
 589 static void
 590 progress_proc (void *cls,
 591                uint64_t offset,
 592                const void *pt_block,
 593                size_t pt_size,
 594                unsigned int depth)
 595 {
 596   struct GNUNET_FS_PublishContext *sc = cls;
 597   struct GNUNET_FS_FileInformation *p;
 598   struct GNUNET_FS_ProgressInfo pi;
 599
 600   p = sc->fi_pos;
 601   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 602   pi.value.publish.specifics.progress.data = pt_block;
 603   pi.value.publish.specifics.progress.offset = offset;
 604   pi.value.publish.specifics.progress.data_len = pt_size;
 605   pi.value.publish.specifics.progress.depth = depth;
 606   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset);
 607 }
 608
 609
 610 /**
 611  * We are uploading a file or directory; load (if necessary) the next
 612  * block into memory, encrypt it and send it to the FS service.  Then
 613  * continue with the main task.
 614  *
 615  * @param sc overall upload data
 616  */
 617 static void
 618 publish_content (struct GNUNET_FS_PublishContext *sc)
 619 {
 620   struct GNUNET_FS_FileInformation *p;
 621   char *emsg;
 622   struct GNUNET_FS_DirectoryBuilder *db;
 623   struct GNUNET_FS_FileInformation *dirpos;
 624   void *raw_data;
 625   uint64_t size;
 626
 627   p = sc->fi_pos;
 628   GNUNET_assert (p != NULL);
 629   if (NULL == p->te)
 630     {
 631       if (p->is_directory)
 632         {
 633 #if DEBUG_PUBLISH
 634           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 635                       "Creating directory\n");
 636 #endif
 637           db = GNUNET_FS_directory_builder_create (p->meta);
 638           dirpos = p->data.dir.entries;
 639           while (NULL != dirpos)
 640             {
 641               if (dirpos->is_directory)
 642                 {
 643                   raw_data = dirpos->data.dir.dir_data;
 644                   dirpos->data.dir.dir_data = NULL;
 645                 }
 646               else
 647                 {
 648                   raw_data = NULL;
 649                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 650                        (dirpos->data.file.file_size > 0) )
 651                     {
 652                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 653                       emsg = NULL;
 654                       if (dirpos->data.file.file_size !=
 655                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 656                                                     0,
 657                                                     dirpos->data.file.file_size,
 658                                                     raw_data,
 659                                                     &emsg))
 660                         {
 661                           GNUNET_free_non_null (emsg);
 662                           GNUNET_free (raw_data);
 663                           raw_data = NULL;
 664                         }
 665                     }
 666                 }
 667               GNUNET_FS_directory_builder_add (db,
 668                                                dirpos->chk_uri,
 669                                                dirpos->meta,
 670                                                raw_data);
 671               GNUNET_free_non_null (raw_data);
 672               dirpos = dirpos->next;
 673             }
 674           GNUNET_FS_directory_builder_finish (db,
 675                                               &p->data.dir.dir_size,
 676                                               &p->data.dir.dir_data);
 677           GNUNET_FS_file_information_sync_ (p);
 678         }
 679       size = (p->is_directory)
 680         ? p->data.dir.dir_size
 681         : p->data.file.file_size;
 682 #if DEBUG_PUBLISH
 683       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 684                   "Creating tree encoder\n");
 685 #endif
 686       p->te = GNUNET_FS_tree_encoder_create (sc->h,
 687                                              size,
 688                                              sc,
 689                                              &block_reader,
 690                                              &block_proc,
 691                                              &progress_proc,
 692                                              &encode_cont);
 693
 694     }
 695 #if DEBUG_PUBLISH
 696   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 697               "Processing next block from tree\n");
 698 #endif
 699   GNUNET_FS_tree_encoder_next (p->te);
 700 }
 701
 702
 703 /**
 704  * Process the response (or lack thereof) from
 705  * the "fs" service to our 'start index' request.
 706  *
 707  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 708  * @param msg the response we got
 709  */
 710 static void
 711 process_index_start_response (void *cls,
 712                               const struct GNUNET_MessageHeader *msg)
 713 {
 714   struct GNUNET_FS_PublishContext *sc = cls;
 715   struct GNUNET_FS_FileInformation *p;
 716   const char *emsg;
 717   uint16_t msize;
 718
 719   GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO);
 720   sc->client = NULL;
 721   p = sc->fi_pos;
 722   if (msg == NULL)
 723     {
 724       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 725                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 726                   p->filename,
 727                   _("timeout on index-start request to `fs' service"));
 728       p->data.file.do_index = GNUNET_NO;
 729       GNUNET_FS_file_information_sync_ (p);
 730       publish_content (sc);
 731       return;
 732     }
 733   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 734     {
 735       msize = ntohs (msg->size);
 736       emsg = (const char *) &msg[1];
 737       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 738            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 739         emsg = gettext_noop ("unknown error");
 740       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 741                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 742                   p->filename,
 743                   gettext (emsg));
 744       p->data.file.do_index = GNUNET_NO;
 745       GNUNET_FS_file_information_sync_ (p);
 746       publish_content (sc);
 747       return;
 748     }
 749   p->data.file.index_start_confirmed = GNUNET_YES;
 750   /* success! continue with indexing */
 751   GNUNET_FS_file_information_sync_ (p);
 752   publish_content (sc);
 753 }
 754
 755
 756 /**
 757  * Function called once the hash computation over an
 758  * indexed file has completed.
 759  *
 760  * @param cls closure, our publishing context
 761  * @param res resulting hash, NULL on error
 762  */
 763 static void
 764 hash_for_index_cb (void *cls,
 765                    const GNUNET_HashCode *
 766                    res)
 767 {
 768   struct GNUNET_FS_PublishContext *sc = cls;
 769   struct GNUNET_FS_FileInformation *p;
 770   struct IndexStartMessage *ism;
 771   size_t slen;
 772   struct GNUNET_CLIENT_Connection *client;
 773   uint64_t dev;
 774   uint64_t ino;
 775   char *fn;
 776
 777   p = sc->fi_pos;
 778   if (NULL == res)
 779     {
 780       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 781                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 782                   p->filename,
 783                   _("failed to compute hash"));
 784       p->data.file.do_index = GNUNET_NO;
 785       GNUNET_FS_file_information_sync_ (p);
 786       publish_content (sc);
 787       return;
 788     }
 789   if (GNUNET_YES == p->data.file.index_start_confirmed)
 790     {
 791       publish_content (sc);
 792       return;
 793     }
 794   fn = GNUNET_STRINGS_filename_expand (p->filename);
 795   GNUNET_assert (fn != NULL);
 796   slen = strlen (fn) + 1;
 797   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 798     {
 799       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 800                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 801                   fn,
 802                   _("filename too long"));
 803       GNUNET_free (fn);
 804       p->data.file.do_index = GNUNET_NO;
 805       GNUNET_FS_file_information_sync_ (p);
 806       publish_content (sc);
 807       return;
 808     }
 809 #if DEBUG_PUBLISH
 810   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 811               "Hash of indexed file `%s' is `%s'\n",
 812               p->filename,
 813               GNUNET_h2s (res));
 814 #endif
 815   client = GNUNET_CLIENT_connect (sc->h->sched,
 816                                   "fs",
 817                                   sc->h->cfg);
 818   if (NULL == client)
 819     {
 820       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 821                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 822                   p->filename,
 823                   _("could not connect to `fs' service"));
 824       p->data.file.do_index = GNUNET_NO;
 825       publish_content (sc);
 826       GNUNET_free (fn);
 827       return;
 828     }
 829   if (p->data.file.have_hash != GNUNET_YES)
 830     {
 831       p->data.file.file_id = *res;
 832       p->data.file.have_hash = GNUNET_YES;
 833       GNUNET_FS_file_information_sync_ (p);
 834     }
 835   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 836                        slen);
 837   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 838                            slen);
 839   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 840   if (GNUNET_OK ==
 841       GNUNET_DISK_file_get_identifiers (p->filename,
 842                                         &dev,
 843                                         &ino))
 844     {
 845       ism->device = GNUNET_htonll (dev);
 846       ism->inode = GNUNET_htonll(ino);
 847     }
 848 #if DEBUG_PUBLISH
 849   else
 850     {
 851       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 852                   _("Failed to get file identifiers for `%s'\n"),
 853                   p->filename);
 854     }
 855 #endif
 856   ism->file_id = *res;
 857   memcpy (&ism[1],
 858           fn,
 859           slen);
 860   GNUNET_free (fn);
 861   sc->client = client;
 862   GNUNET_break (GNUNET_YES ==
 863                 GNUNET_CLIENT_transmit_and_get_response (client,
 864                                                          &ism->header,
 865                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 866                                                          GNUNET_YES,
 867                                                          &process_index_start_response,
 868                                                          sc));
 869   GNUNET_free (ism);
 870 }
 871
 872
 873 /**
 874  * Main function that performs the upload.
 875  *
 876  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 877  * @param tc task context
 878  */
 879 void
 880 GNUNET_FS_publish_main_ (void *cls,
 881                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 882 {
 883   struct GNUNET_FS_PublishContext *pc = cls;
 884   struct GNUNET_FS_ProgressInfo pi;
 885   struct GNUNET_FS_FileInformation *p;
 886   char *fn;
 887
 888   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 889   p = pc->fi_pos;
 890   if (NULL == p)
 891     {
 892 #if DEBUG_PUBLISH
 893       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 894                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 895 #endif
 896       /* upload of entire hierarchy complete,
 897          publish namespace entries */
 898       GNUNET_FS_publish_sync_ (pc);
 899       publish_sblock (pc);
 900       return;
 901     }
 902   /* find starting position */
 903   while ( (p->is_directory) &&
 904           (NULL != p->data.dir.entries) &&
 905           (NULL == p->emsg) &&
 906           (NULL == p->data.dir.entries->chk_uri) )
 907     {
 908       p = p->data.dir.entries;
 909       pc->fi_pos = p;
 910       GNUNET_FS_publish_sync_ (pc);
 911     }
 912   /* abort on error */
 913   if (NULL != p->emsg)
 914     {
 915 #if DEBUG_PUBLISH
 916       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 917                   "Error uploading: %s\n",
 918                   p->emsg);
 919 #endif
 920       /* error with current file, abort all
 921          related files as well! */
 922       while (NULL != p->dir)
 923         {
 924           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 925                                                        EXTRACTOR_METATYPE_FILENAME);
 926           p = p->dir;
 927           if (fn != NULL)
 928             {
 929               GNUNET_asprintf (&p->emsg,
 930                                _("Recursive upload failed at `%s': %s"),
 931                                fn,
 932                                p->emsg);
 933               GNUNET_free (fn);
 934             }
 935           else
 936             {
 937               GNUNET_asprintf (&p->emsg,
 938                                _("Recursive upload failed: %s"),
 939                                p->emsg);
 940             }
 941           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 942           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 943           pi.value.publish.specifics.error.message = p->emsg;
 944           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 945         }
 946       pc->all_done = GNUNET_YES;
 947       GNUNET_FS_publish_sync_ (pc);
 948       return;
 949     }
 950   /* handle completion */
 951   if (NULL != p->chk_uri)
 952     {
 953 #if DEBUG_PUBLISH
 954       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 955                   "File upload complete, now publishing KSK blocks.\n");
 956 #endif
 957       GNUNET_FS_publish_sync_ (pc);
 958       /* upload of "p" complete, publish KBlocks! */
 959       if (p->keywords != NULL)
 960         {
 961           GNUNET_FS_publish_ksk (pc->h,
 962                                  p->keywords,
 963                                  p->meta,
 964                                  p->chk_uri,
 965                                  p->expirationTime,
 966                                  p->anonymity,
 967                                  p->priority,
 968                                  pc->options,
 969                                  &publish_kblocks_cont,
 970                                  pc);
 971         }
 972       else
 973         {
 974           publish_kblocks_cont (pc,
 975                                 p->chk_uri,
 976                                 NULL);
 977         }
 978       return;
 979     }
 980   if ( (!p->is_directory) &&
 981        (p->data.file.do_index) )
 982     {
 983       if (NULL == p->filename)
 984         {
 985           p->data.file.do_index = GNUNET_NO;
 986           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 987                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 988                       "<no-name>",
 989                       _("needs to be an actual file"));
 990           GNUNET_FS_file_information_sync_ (p);
 991           publish_content (pc);
 992           return;
 993         }
 994       if (p->data.file.have_hash)
 995         {
 996           hash_for_index_cb (pc,
 997                              &p->data.file.file_id);
 998         }
 999       else
1000         {
1001           p->start_time = GNUNET_TIME_absolute_get ();
1002           GNUNET_CRYPTO_hash_file (pc->h->sched,
1003                                    GNUNET_SCHEDULER_PRIORITY_IDLE,
1004                                    p->filename,
1005                                    HASHING_BLOCKSIZE,
1006                                    &hash_for_index_cb,
1007                                    pc);
1008         }
1009       return;
1010     }
1011   publish_content (pc);
1012 }
1013
1014
1015 /**
1016  * Signal the FS's progress function that we are starting
1017  * an upload.
1018  *
1019  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1020  * @param fi the entry in the publish-structure
1021  * @param length length of the file or directory
1022  * @param meta metadata for the file or directory (can be modified)
1023  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1024  * @param anonymity pointer to selected anonymity level (can be modified)
1025  * @param priority pointer to selected priority (can be modified)
1026  * @param do_index should we index?
1027  * @param expirationTime pointer to selected expiration time (can be modified)
1028  * @param client_info pointer to client context set upon creation (can be modified)
1029  * @return GNUNET_OK to continue (always)
1030  */
1031 static int
1032 fip_signal_start(void *cls,
1033                  struct GNUNET_FS_FileInformation *fi,
1034                  uint64_t length,
1035                  struct GNUNET_CONTAINER_MetaData *meta,
1036                  struct GNUNET_FS_Uri **uri,
1037                  uint32_t *anonymity,
1038                  uint32_t *priority,
1039                  int *do_index,
1040                  struct GNUNET_TIME_Absolute *expirationTime,
1041                  void **client_info)
1042 {
1043   struct GNUNET_FS_PublishContext *sc = cls;
1044   struct GNUNET_FS_ProgressInfo pi;
1045   unsigned int kc;
1046   uint64_t left;
1047
1048   if (*do_index)
1049     {
1050       /* space for on-demand blocks */
1051       sc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1052     }
1053   else
1054     {
1055       /* space for DBlocks */
1056       sc->reserve_space += length;
1057     }
1058   /* entries for IBlocks and DBlocks, space for IBlocks */
1059   left = length;
1060   while (1)
1061     {
1062       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1063       sc->reserve_entries += left;
1064       if (left == 1)
1065         break;
1066       left = left * sizeof (struct ContentHashKey);
1067       sc->reserve_space += left;
1068     }
1069   /* entries and space for keywords */
1070   if (NULL != *uri)
1071     {
1072       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1073       sc->reserve_entries += kc;
1074       sc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1075     }
1076   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1077   *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
1078   GNUNET_FS_file_information_sync_ (fi);
1079   return GNUNET_OK;
1080 }
1081
1082
1083 /**
1084  * Signal the FS's progress function that we are suspending
1085  * an upload.
1086  *
1087  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1088  * @param fi the entry in the publish-structure
1089  * @param length length of the file or directory
1090  * @param meta metadata for the file or directory (can be modified)
1091  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1092  * @param anonymity pointer to selected anonymity level (can be modified)
1093  * @param priority pointer to selected priority (can be modified)
1094  * @param do_index should we index?
1095  * @param expirationTime pointer to selected expiration time (can be modified)
1096  * @param client_info pointer to client context set upon creation (can be modified)
1097  * @return GNUNET_OK to continue (always)
1098  */
1099 static int
1100 fip_signal_suspend(void *cls,
1101                    struct GNUNET_FS_FileInformation *fi,
1102                    uint64_t length,
1103                    struct GNUNET_CONTAINER_MetaData *meta,
1104                    struct GNUNET_FS_Uri **uri,
1105                    uint32_t *anonymity,
1106                    uint32_t *priority,
1107                    int *do_index,
1108                    struct GNUNET_TIME_Absolute *expirationTime,
1109                    void **client_info)
1110 {
1111   struct GNUNET_FS_PublishContext*sc = cls;
1112   struct GNUNET_FS_ProgressInfo pi;
1113   uint64_t off;
1114
1115   GNUNET_free_non_null (fi->serialization);
1116   fi->serialization = NULL;
1117   off = (fi->chk_uri == NULL) ? 0 : length;
1118   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1119   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1120   *client_info = NULL;
1121   if (NULL != sc->dsh)
1122     {
1123       GNUNET_DATASTORE_disconnect (sc->dsh, GNUNET_NO);
1124       sc->dsh = NULL;
1125     }
1126   return GNUNET_OK;
1127 }
1128
1129
1130 /**
1131  * Create SUSPEND event for the given publish operation
1132  * and then clean up our state (without stop signal).
1133  *
1134  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1135  */
1136 void
1137 GNUNET_FS_publish_signal_suspend_ (void *cls)
1138 {
1139   struct GNUNET_FS_PublishContext *pc = cls;
1140
1141   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1142     {
1143       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1144       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1145     }
1146   GNUNET_FS_file_information_inspect (pc->fi,
1147                                       &fip_signal_suspend,
1148                                       pc);
1149   GNUNET_FS_end_top (pc->h, pc->top);
1150   publish_cleanup (pc, NULL);
1151 }
1152
1153
1154 /**
1155  * We have gotten a reply for our space reservation request.
1156  * Either fail (insufficient space) or start publishing for good.
1157  *
1158  * @param cls the 'struct GNUNET_FS_PublishContext*'
1159  * @param success positive reservation ID on success
1160  * @param msg error message on error, otherwise NULL
1161  */
1162 static void
1163 finish_reserve (void *cls,
1164                 int success,
1165                 const char *msg)
1166 {
1167   struct GNUNET_FS_PublishContext *pc = cls;
1168
1169   pc->qre = NULL;
1170   if ( (msg != NULL) ||
1171        (success <= 0) )
1172     {
1173       GNUNET_asprintf (&pc->fi->emsg,
1174                        _("Insufficient space for publishing: %s"),
1175                        msg);
1176       signal_publish_error (pc->fi,
1177                             pc,
1178                             pc->fi->emsg);
1179       return;
1180     }
1181   pc->rid = success;
1182   pc->upload_task
1183     = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
1184                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1185                                           &GNUNET_FS_publish_main_,
1186                                           pc);
1187 }
1188
1189
1190 /**
1191  * Publish a file or directory.
1192  *
1193  * @param h handle to the file sharing subsystem
1194  * @param fi information about the file or directory structure to publish
1195  * @param namespace namespace to publish the file in, NULL for no namespace
1196  * @param nid identifier to use for the publishd content in the namespace
1197  *        (can be NULL, must be NULL if namespace is NULL)
1198  * @param nuid update-identifier that will be used for future updates
1199  *        (can be NULL, must be NULL if namespace or nid is NULL)
1200  * @param options options for the publication
1201  * @return context that can be used to control the publish operation
1202  */
1203 struct GNUNET_FS_PublishContext *
1204 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1205                          struct GNUNET_FS_FileInformation *fi,
1206                          struct GNUNET_FS_Namespace *namespace,
1207                          const char *nid,
1208                          const char *nuid,
1209                          enum GNUNET_FS_PublishOptions options)
1210 {
1211   struct GNUNET_FS_PublishContext *ret;
1212   struct GNUNET_DATASTORE_Handle *dsh;
1213
1214   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1215     {
1216       dsh = GNUNET_DATASTORE_connect (h->cfg,
1217                                       h->sched);
1218       if (NULL == dsh)
1219         return NULL;
1220     }
1221   else
1222     {
1223       dsh = NULL;
1224     }
1225   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1226   ret->dsh = dsh;
1227   ret->h = h;
1228   ret->fi = fi;
1229   ret->namespace = namespace;
1230   if (namespace != NULL)
1231     {
1232       namespace->rc++;
1233       GNUNET_assert (NULL != nid);
1234       ret->nid = GNUNET_strdup (nid);
1235       if (NULL != nuid)
1236         ret->nuid = GNUNET_strdup (nuid);
1237     }
1238   /* signal start */
1239   GNUNET_FS_file_information_inspect (ret->fi,
1240                                       &fip_signal_start,
1241                                       ret);
1242   ret->fi_pos = ret->fi;
1243   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1244   GNUNET_FS_publish_sync_ (ret);
1245   if (NULL != ret->dsh)
1246     {
1247       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1248                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1249                   (unsigned int) ret->reserve_entries,
1250                   (unsigned long long) ret->reserve_space);
1251       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1252                                            ret->reserve_space,
1253                                            ret->reserve_entries,
1254                                            UINT_MAX,
1255                                            UINT_MAX,
1256                                            GNUNET_TIME_UNIT_FOREVER_REL,
1257                                            &finish_reserve,
1258                                            ret);
1259     }
1260   else
1261     {
1262       ret->upload_task
1263         = GNUNET_SCHEDULER_add_with_priority (h->sched,
1264                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1265                                               &GNUNET_FS_publish_main_,
1266                                               ret);
1267     }
1268   return ret;
1269 }
1270
1271
1272 /**
1273  * Signal the FS's progress function that we are stopping
1274  * an upload.
1275  *
1276  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1277  * @param fi the entry in the publish-structure
1278  * @param length length of the file or directory
1279  * @param meta metadata for the file or directory (can be modified)
1280  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1281  * @param anonymity pointer to selected anonymity level (can be modified)
1282  * @param priority pointer to selected priority (can be modified)
1283  * @param do_index should we index?
1284  * @param expirationTime pointer to selected expiration time (can be modified)
1285  * @param client_info pointer to client context set upon creation (can be modified)
1286  * @return GNUNET_OK to continue (always)
1287  */
1288 static int
1289 fip_signal_stop(void *cls,
1290                 struct GNUNET_FS_FileInformation *fi,
1291                 uint64_t length,
1292                 struct GNUNET_CONTAINER_MetaData *meta,
1293                 struct GNUNET_FS_Uri **uri,
1294                 uint32_t *anonymity,
1295                 uint32_t *priority,
1296                 int *do_index,
1297                 struct GNUNET_TIME_Absolute *expirationTime,
1298                 void **client_info)
1299 {
1300   struct GNUNET_FS_PublishContext*sc = cls;
1301   struct GNUNET_FS_ProgressInfo pi;
1302   uint64_t off;
1303
1304   if (fi->serialization != NULL)
1305     {
1306       GNUNET_FS_remove_sync_file_ (sc->h,
1307                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1308                                    fi->serialization);
1309       GNUNET_free (fi->serialization);
1310       fi->serialization = NULL;
1311     }
1312   off = (fi->chk_uri == NULL) ? 0 : length;
1313   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1314   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1315   *client_info = NULL;
1316   return GNUNET_OK;
1317 }
1318
1319
1320 /**
1321  * Stop an upload.  Will abort incomplete uploads (but
1322  * not remove blocks that have already been publishd) or
1323  * simply clean up the state for completed uploads.
1324  * Must NOT be called from within the event callback!
1325  *
1326  * @param pc context for the upload to stop
1327  */
1328 void
1329 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1330 {
1331   GNUNET_FS_end_top (pc->h, pc->top);
1332   if (NULL != pc->qre)
1333     {
1334       GNUNET_DATASTORE_cancel (pc->qre);
1335       pc->qre = NULL;
1336     }
1337   if (NULL != pc->dsh)
1338     {
1339       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1340       pc->dsh = NULL;
1341     }
1342   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1343     {
1344       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1345       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1346     }
1347   if (pc->serialization != NULL)
1348     {
1349       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1350       GNUNET_free (pc->serialization);
1351       pc->serialization = NULL;
1352     }
1353   GNUNET_FS_file_information_inspect (pc->fi,
1354                                       &fip_signal_stop,
1355                                       pc);
1356   if (GNUNET_YES == pc->in_network_wait)
1357     {
1358       pc->in_network_wait = GNUNET_SYSERR;
1359       return;
1360     }
1361   publish_cleanup (pc, NULL);
1362 }
1363
1364
1365 /**
1366  * Context for the KSK publication.
1367  */
1368 struct PublishKskContext
1369 {
1370
1371   /**
1372    * Keywords to use.
1373    */
1374   struct GNUNET_FS_Uri *ksk_uri;
1375
1376   /**
1377    * Global FS context.
1378    */
1379   struct GNUNET_FS_Handle *h;
1380
1381   /**
1382    * The master block that we are sending
1383    * (in plaintext), has "mdsize+slen" more
1384    * bytes than the struct would suggest.
1385    */
1386   struct KBlock *kb;
1387
1388   /**
1389    * Buffer of the same size as "kb" for
1390    * the encrypted version.
1391    */
1392   struct KBlock *cpy;
1393
1394   /**
1395    * Handle to the datastore, NULL if we are just
1396    * simulating.
1397    */
1398   struct GNUNET_DATASTORE_Handle *dsh;
1399
1400   /**
1401    * Function to call once we're done.
1402    */
1403   GNUNET_FS_PublishContinuation cont;
1404
1405   /**
1406    * Closure for cont.
1407    */
1408   void *cont_cls;
1409
1410   /**
1411    * When should the KBlocks expire?
1412    */
1413   struct GNUNET_TIME_Absolute expirationTime;
1414
1415   /**
1416    * Size of the serialized metadata.
1417    */
1418   ssize_t mdsize;
1419
1420   /**
1421    * Size of the (CHK) URI as a string.
1422    */
1423   size_t slen;
1424
1425   /**
1426    * Keyword that we are currently processing.
1427    */
1428   unsigned int i;
1429
1430   /**
1431    * Anonymity level for the KBlocks.
1432    */
1433   uint32_t anonymity;
1434
1435   /**
1436    * Priority for the KBlocks.
1437    */
1438   uint32_t priority;
1439 };
1440
1441
1442 /**
1443  * Continuation of "GNUNET_FS_publish_ksk" that performs
1444  * the actual publishing operation (iterating over all
1445  * of the keywords).
1446  *
1447  * @param cls closure of type "struct PublishKskContext*"
1448  * @param tc unused
1449  */
1450 static void
1451 publish_ksk_cont (void *cls,
1452                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1453
1454
1455 /**
1456  * Function called by the datastore API with
1457  * the result from the PUT request.
1458  *
1459  * @param cls closure of type "struct PublishKskContext*"
1460  * @param success GNUNET_OK on success
1461  * @param msg error message (or NULL)
1462  */
1463 static void
1464 kb_put_cont (void *cls,
1465              int success,
1466              const char *msg)
1467 {
1468   struct PublishKskContext *pkc = cls;
1469
1470   if (GNUNET_OK != success)
1471     {
1472       if (NULL != pkc->dsh)
1473         {
1474           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1475           pkc->dsh = NULL;
1476         }
1477       GNUNET_free (pkc->cpy);
1478       GNUNET_free (pkc->kb);
1479       pkc->cont (pkc->cont_cls,
1480                  NULL,
1481                  msg);
1482       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1483       GNUNET_free (pkc);
1484       return;
1485     }
1486   GNUNET_SCHEDULER_add_continuation (pkc->h->sched,
1487                                      &publish_ksk_cont,
1488                                      pkc,
1489                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1490 }
1491
1492
1493 /**
1494  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1495  * publishing operation (iterating over all of the keywords).
1496  *
1497  * @param cls closure of type "struct PublishKskContext*"
1498  * @param tc unused
1499  */
1500 static void
1501 publish_ksk_cont (void *cls,
1502                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1503 {
1504   struct PublishKskContext *pkc = cls;
1505   const char *keyword;
1506   GNUNET_HashCode key;
1507   GNUNET_HashCode query;
1508   struct GNUNET_CRYPTO_AesSessionKey skey;
1509   struct GNUNET_CRYPTO_AesInitializationVector iv;
1510   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1511
1512
1513   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1514        (NULL == pkc->dsh) )
1515     {
1516       if (NULL != pkc->dsh)
1517         {
1518           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1519           pkc->dsh = NULL;
1520         }
1521       GNUNET_free (pkc->cpy);
1522       GNUNET_free (pkc->kb);
1523       pkc->cont (pkc->cont_cls,
1524                  pkc->ksk_uri,
1525                  NULL);
1526       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1527       GNUNET_free (pkc);
1528       return;
1529     }
1530   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1531   /* first character of keyword indicates if it is
1532      mandatory or not -- ignore for hashing */
1533   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1534   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1535   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1536                              pkc->slen + pkc->mdsize,
1537                              &skey,
1538                              &iv,
1539                              &pkc->cpy[1]);
1540   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1541   GNUNET_assert (NULL != pk);
1542   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1543   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1544                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1545                       &query);
1546   GNUNET_assert (GNUNET_OK ==
1547                  GNUNET_CRYPTO_rsa_sign (pk,
1548                                          &pkc->cpy->purpose,
1549                                          &pkc->cpy->signature));
1550   GNUNET_CRYPTO_rsa_key_free (pk);
1551   GNUNET_DATASTORE_put (pkc->dsh,
1552                         0,
1553                         &query,
1554                         pkc->mdsize +
1555                         sizeof (struct KBlock) +
1556                         pkc->slen,
1557                         pkc->cpy,
1558                         GNUNET_BLOCK_TYPE_KBLOCK,
1559                         pkc->priority,
1560                         pkc->anonymity,
1561                         pkc->expirationTime,
1562                         -2, 1,
1563                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1564                         &kb_put_cont,
1565                         pkc);
1566 }
1567
1568
1569 /**
1570  * Publish a CHK under various keywords on GNUnet.
1571  *
1572  * @param h handle to the file sharing subsystem
1573  * @param ksk_uri keywords to use
1574  * @param meta metadata to use
1575  * @param uri URI to refer to in the KBlock
1576  * @param expirationTime when the KBlock expires
1577  * @param anonymity anonymity level for the KBlock
1578  * @param priority priority for the KBlock
1579  * @param options publication options
1580  * @param cont continuation
1581  * @param cont_cls closure for cont
1582  */
1583 void
1584 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1585                        const struct GNUNET_FS_Uri *ksk_uri,
1586                        const struct GNUNET_CONTAINER_MetaData *meta,
1587                        const struct GNUNET_FS_Uri *uri,
1588                        struct GNUNET_TIME_Absolute expirationTime,
1589                        uint32_t anonymity,
1590                        uint32_t priority,
1591                        enum GNUNET_FS_PublishOptions options,
1592                        GNUNET_FS_PublishContinuation cont,
1593                        void *cont_cls)
1594 {
1595   struct PublishKskContext *pkc;
1596   char *uris;
1597   size_t size;
1598   char *kbe;
1599   char *sptr;
1600
1601   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1602   pkc->h = h;
1603   pkc->expirationTime = expirationTime;
1604   pkc->anonymity = anonymity;
1605   pkc->priority = priority;
1606   pkc->cont = cont;
1607   pkc->cont_cls = cont_cls;
1608   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1609     {
1610       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg,
1611                                            h->sched);
1612       if (pkc->dsh == NULL)
1613         {
1614           cont (cont_cls, NULL, _("Could not connect to datastore."));
1615           GNUNET_free (pkc);
1616           return;
1617         }
1618     }
1619   if (meta == NULL)
1620     pkc->mdsize = 0;
1621   else
1622     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1623   GNUNET_assert (pkc->mdsize >= 0);
1624   uris = GNUNET_FS_uri_to_string (uri);
1625   pkc->slen = strlen (uris) + 1;
1626   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1627   if (size > MAX_KBLOCK_SIZE)
1628     {
1629       size = MAX_KBLOCK_SIZE;
1630       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1631     }
1632   pkc->kb = GNUNET_malloc (size);
1633   kbe = (char *) &pkc->kb[1];
1634   memcpy (kbe, uris, pkc->slen);
1635   GNUNET_free (uris);
1636   sptr = &kbe[pkc->slen];
1637   if (meta != NULL)
1638     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1639                                                         &sptr,
1640                                                         pkc->mdsize,
1641                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1642   if (pkc->mdsize == -1)
1643     {
1644       GNUNET_break (0);
1645       GNUNET_free (pkc->kb);
1646       if (pkc->dsh != NULL)
1647         {
1648           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1649           pkc->dsh = NULL;
1650         }
1651       cont (cont_cls, NULL, _("Internal error."));
1652       GNUNET_free (pkc);
1653       return;
1654     }
1655   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1656
1657   pkc->cpy = GNUNET_malloc (size);
1658   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1659                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1660                                   pkc->mdsize +
1661                                   pkc->slen);
1662   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1663   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1664   GNUNET_SCHEDULER_add_continuation (h->sched,
1665                                      &publish_ksk_cont,
1666                                      pkc,
1667                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1668 }
1669
1670
1671 /**
1672  * Context for the SKS publication.
1673  */
1674 struct PublishSksContext
1675 {
1676
1677   /**
1678    * Global FS context.
1679    */
1680   struct GNUNET_FS_Uri *uri;
1681
1682   /**
1683    * Handle to the datastore.
1684    */
1685   struct GNUNET_DATASTORE_Handle *dsh;
1686
1687   /**
1688    * Function to call once we're done.
1689    */
1690   GNUNET_FS_PublishContinuation cont;
1691
1692   /**
1693    * Closure for cont.
1694    */
1695   void *cont_cls;
1696
1697 };
1698
1699
1700 /**
1701  * Function called by the datastore API with
1702  * the result from the PUT (SBlock) request.
1703  *
1704  * @param cls closure of type "struct PublishSksContext*"
1705  * @param success GNUNET_OK on success
1706  * @param msg error message (or NULL)
1707  */
1708 static void
1709 sb_put_cont (void *cls,
1710              int success,
1711              const char *msg)
1712 {
1713   struct PublishSksContext *psc = cls;
1714
1715   if (NULL != psc->dsh)
1716     {
1717       GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO);
1718       psc->dsh = NULL;
1719     }
1720   if (GNUNET_OK != success)
1721     psc->cont (psc->cont_cls,
1722                NULL,
1723                msg);
1724   else
1725     psc->cont (psc->cont_cls,
1726                psc->uri,
1727                NULL);
1728   GNUNET_FS_uri_destroy (psc->uri);
1729   GNUNET_free (psc);
1730 }
1731
1732
1733 /**
1734  * Publish an SBlock on GNUnet.
1735  *
1736  * @param h handle to the file sharing subsystem
1737  * @param namespace namespace to publish in
1738  * @param identifier identifier to use
1739  * @param update update identifier to use
1740  * @param meta metadata to use
1741  * @param uri URI to refer to in the SBlock
1742  * @param expirationTime when the SBlock expires
1743  * @param anonymity anonymity level for the SBlock
1744  * @param priority priority for the SBlock
1745  * @param options publication options
1746  * @param cont continuation
1747  * @param cont_cls closure for cont
1748  */
1749 void
1750 GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
1751                        struct GNUNET_FS_Namespace *namespace,
1752                        const char *identifier,
1753                        const char *update,
1754                        const struct GNUNET_CONTAINER_MetaData *meta,
1755                        const struct GNUNET_FS_Uri *uri,
1756                        struct GNUNET_TIME_Absolute expirationTime,
1757                        uint32_t anonymity,
1758                        uint32_t priority,
1759                        enum GNUNET_FS_PublishOptions options,
1760                        GNUNET_FS_PublishContinuation cont,
1761                        void *cont_cls)
1762 {
1763   struct PublishSksContext *psc;
1764   struct GNUNET_CRYPTO_AesSessionKey sk;
1765   struct GNUNET_CRYPTO_AesInitializationVector iv;
1766   struct GNUNET_FS_Uri *sks_uri;
1767   char *uris;
1768   size_t size;
1769   size_t slen;
1770   size_t nidlen;
1771   size_t idlen;
1772   ssize_t mdsize;
1773   struct SBlock *sb;
1774   struct SBlock *sb_enc;
1775   char *dest;
1776   struct GNUNET_CONTAINER_MetaData *mmeta;
1777   GNUNET_HashCode key;         /* hash of thisId = key */
1778   GNUNET_HashCode id;          /* hash of hc = identifier */
1779   GNUNET_HashCode query;       /* id ^ nsid = DB query */
1780
1781   if (NULL == meta)
1782     mmeta = GNUNET_CONTAINER_meta_data_create ();
1783   else
1784     mmeta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1785   uris = GNUNET_FS_uri_to_string (uri);
1786   slen = strlen (uris) + 1;
1787   idlen = strlen (identifier);
1788   if (update == NULL)
1789     update = "";
1790   nidlen = strlen (update) + 1;
1791   mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (mmeta);
1792   size = sizeof (struct SBlock) + slen + nidlen + mdsize;
1793   if (size > MAX_SBLOCK_SIZE)
1794     {
1795       size = MAX_SBLOCK_SIZE;
1796       mdsize = size - (sizeof (struct SBlock) + slen + nidlen);
1797     }
1798   sb = GNUNET_malloc (sizeof (struct SBlock) + size);
1799   dest = (char *) &sb[1];
1800   memcpy (dest, update, nidlen);
1801   dest += nidlen;
1802   memcpy (dest, uris, slen);
1803   GNUNET_free (uris);
1804   dest += slen;
1805   mdsize = GNUNET_CONTAINER_meta_data_serialize (mmeta,
1806                                                  &dest,
1807                                                  mdsize,
1808                                                  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1809   GNUNET_CONTAINER_meta_data_destroy (mmeta);
1810   if (mdsize == -1)
1811     {
1812       GNUNET_break (0);
1813       GNUNET_free (sb);
1814       cont (cont_cls,
1815             NULL,
1816             _("Internal error."));
1817       return;
1818     }
1819   size = sizeof (struct SBlock) + mdsize + slen + nidlen;
1820   sb_enc = GNUNET_malloc (size);
1821   GNUNET_CRYPTO_hash (identifier, idlen, &key);
1822   GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &id);
1823   sks_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri));
1824   sks_uri->type = sks;
1825   GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &sb_enc->subspace);
1826   GNUNET_CRYPTO_hash (&sb_enc->subspace,
1827                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1828                       &sks_uri->data.sks.namespace);
1829   sks_uri->data.sks.identifier = GNUNET_strdup (identifier);
1830   GNUNET_CRYPTO_hash_xor (&id,
1831                           &sks_uri->data.sks.namespace,
1832                           &sb_enc->identifier);
1833   GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
1834   GNUNET_CRYPTO_aes_encrypt (&sb[1],
1835                              size - sizeof (struct SBlock),
1836                              &sk,
1837                              &iv,
1838                              &sb_enc[1]);
1839   sb_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK);
1840   sb_enc->purpose.size = htonl(slen + mdsize + nidlen
1841                                + sizeof(struct SBlock)
1842                                - sizeof(struct GNUNET_CRYPTO_RsaSignature));
1843   GNUNET_assert (GNUNET_OK ==
1844                  GNUNET_CRYPTO_rsa_sign (namespace->key,
1845                                          &sb_enc->purpose,
1846                                          &sb_enc->signature));
1847   psc = GNUNET_malloc (sizeof(struct PublishSksContext));
1848   psc->uri = sks_uri;
1849   psc->cont = cont;
1850   psc->cont_cls = cont_cls;
1851   if (0 != (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1852     {
1853       GNUNET_free (sb_enc);
1854       GNUNET_free (sb);
1855       sb_put_cont (psc,
1856                    GNUNET_OK,
1857                    NULL);
1858       return;
1859     }
1860   psc->dsh = GNUNET_DATASTORE_connect (h->cfg, h->sched);
1861   if (NULL == psc->dsh)
1862     {
1863       GNUNET_free (sb_enc);
1864       GNUNET_free (sb);
1865       sb_put_cont (psc,
1866                    GNUNET_NO,
1867                    _("Failed to connect to datastore."));
1868       return;
1869     }
1870   GNUNET_CRYPTO_hash_xor (&sks_uri->data.sks.namespace,
1871                           &id,
1872                           &query);
1873   GNUNET_DATASTORE_put (psc->dsh,
1874                         0,
1875                         &sb_enc->identifier,
1876                         size,
1877                         sb_enc,
1878                         GNUNET_BLOCK_TYPE_SBLOCK,
1879                         priority,
1880                         anonymity,
1881                         expirationTime,
1882                         -2, 1,
1883                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1884                         &sb_put_cont,
1885                         psc);
1886   GNUNET_free (sb);
1887   GNUNET_free (sb_enc);
1888 }
1889
1890 /* end of fs_publish.c */