src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see http://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  *
  28  * TODO:
  29  * - indexing cleanup: unindex on failure (can wait)
  30  * - datastore reservation support (optimization)
  31  * - location URIs (publish with anonymity-level zero)
  32  */
  33
  34 #include "platform.h"
  35 #include "gnunet_constants.h"
  36 #include "gnunet_signatures.h"
  37 #include "gnunet_util_lib.h"
  38 #include "gnunet_fs_service.h"
  39 #include "fs.h"
  40 #include "fs_tree.h"
  41
  42 #define DEBUG_PUBLISH GNUNET_NO
  43
  44
  45 /**
  46  * Context for "ds_put_cont".
  47  */
  48 struct PutContCtx
  49 {
  50   /**
  51    * Current publishing context.
  52    */
  53   struct GNUNET_FS_PublishContext *sc;
  54
  55   /**
  56    * Specific file with the block.
  57    */
  58   struct GNUNET_FS_FileInformation *p;
  59
  60   /**
  61    * Function to run next, if any (can be NULL).
  62    */
  63   GNUNET_SCHEDULER_Task cont;
  64
  65   /**
  66    * Closure for cont.
  67    */
  68   void *cont_cls;
  69 };
  70
  71
  72 /**
  73  * Fill in all of the generic fields for
  74  * a publish event and call the callback.
  75  *
  76  * @param pi structure to fill in
  77  * @param sc overall publishing context
  78  * @param p file information for the file being published
  79  * @param offset where in the file are we so far
  80  * @return value returned from callback
  81  */
  82 void *
  83 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  84                                 struct GNUNET_FS_PublishContext *sc,
  85                                 const struct GNUNET_FS_FileInformation *p,
  86                                 uint64_t offset)
  87 {
  88   pi->value.publish.pc = sc;
  89   pi->value.publish.fi = p;
  90   pi->value.publish.cctx
  91     = p->client_info;
  92   pi->value.publish.pctx
  93     = (NULL == p->dir) ? NULL : p->dir->client_info;
  94   pi->value.publish.filename = p->filename;
  95   pi->value.publish.size
  96     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  97   pi->value.publish.eta
  98     = GNUNET_TIME_calculate_eta (p->start_time,
  99                                  offset,
 100                                  pi->value.publish.size);
 101   pi->value.publish.completed = offset;
 102   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
 103   pi->value.publish.anonymity = p->anonymity;
 104   return sc->h->upcb (sc->h->upcb_cls,
 105                       pi);
 106 }
 107
 108
 109 /**
 110  * Cleanup the publish context, we're done with it.
 111  *
 112  * @param cls struct to clean up after
 113  * @param tc scheduler context
 114  */
 115 static void
 116 publish_cleanup (void *cls,
 117                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 118 {
 119   struct GNUNET_FS_PublishContext *pc = cls;
 120
 121   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 122   if (pc->namespace != NULL)
 123     GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 124   GNUNET_free_non_null (pc->nid);
 125   GNUNET_free_non_null (pc->nuid);
 126   GNUNET_free_non_null (pc->serialization);
 127   if (pc->dsh != NULL)
 128     {
 129       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 130       pc->dsh = NULL;
 131     }
 132   if (pc->client != NULL)
 133     GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 134   GNUNET_free (pc);
 135 }
 136
 137
 138 /**
 139  * Function called by the datastore API with
 140  * the result from the PUT request.
 141  *
 142  * @param cls our closure
 143  * @param success GNUNET_OK on success
 144  * @param msg error message (or NULL)
 145  */
 146 static void
 147 ds_put_cont (void *cls,
 148              int success,
 149              const char *msg)
 150 {
 151   struct PutContCtx *pcc = cls;
 152   struct GNUNET_FS_ProgressInfo pi;
 153
 154   if (GNUNET_SYSERR == pcc->sc->in_network_wait)
 155     {
 156       /* we were aborted in the meantime, finish shutdown! */
 157       GNUNET_SCHEDULER_add_continuation (pcc->sc->h->sched,
 158                                          &publish_cleanup,
 159                                          pcc->sc,
 160                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 161       GNUNET_free (pcc);
 162       return;
 163     }
 164   GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait);
 165   pcc->sc->in_network_wait = GNUNET_NO;
 166   if (GNUNET_OK != success)
 167     {
 168       GNUNET_asprintf (&pcc->p->emsg,
 169                        _("Publishing failed: %s"),
 170                        msg);
 171       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 172       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 173       pi.value.publish.specifics.error.message = pcc->p->emsg;
 174       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0);
 175     }
 176   if (NULL != pcc->cont)
 177     pcc->sc->upload_task
 178       = GNUNET_SCHEDULER_add_with_priority (pcc->sc->h->sched,
 179                                             GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 180                                             pcc->cont,
 181                                             pcc->cont_cls);
 182   GNUNET_free (pcc);
 183 }
 184
 185
 186 /**
 187  * Generate the callback that signals clients
 188  * that a file (or directory) has been completely
 189  * published.
 190  *
 191  * @param p the completed upload
 192  * @param sc context of the publication
 193  */
 194 static void
 195 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 196                            struct GNUNET_FS_PublishContext *sc)
 197 {
 198   struct GNUNET_FS_ProgressInfo pi;
 199
 200   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 201   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 202   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 203   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 204                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 205 }
 206
 207
 208 /**
 209  * Generate the callback that signals clients
 210  * that a file (or directory) has encountered
 211  * a problem during publication.
 212  *
 213  * @param p the upload that had trouble
 214  * @param sc context of the publication
 215  * @param emsg error message
 216  */
 217 static void
 218 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 219                       struct GNUNET_FS_PublishContext *sc,
 220                       const char *emsg)
 221 {
 222   struct GNUNET_FS_ProgressInfo pi;
 223
 224   p->emsg = GNUNET_strdup (emsg);
 225   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 226   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 227   pi.value.publish.specifics.error.message =emsg;
 228   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 229 }
 230
 231
 232 /**
 233  * Datastore returns from reservation cancel request.
 234  *
 235  * @param cls the 'struct GNUNET_FS_PublishContext'
 236  * @param success success code (not used)
 237  * @param msg error message (typically NULL, not used)
 238  */
 239 static void
 240 finish_release_reserve (void *cls,
 241                         int success,
 242                         const char *msg)
 243 {
 244   struct GNUNET_FS_PublishContext *pc = cls;
 245
 246   pc->qre = NULL;
 247   signal_publish_completion (pc->fi, pc);
 248   pc->all_done = GNUNET_YES;
 249   GNUNET_FS_publish_sync_ (pc);
 250 }
 251
 252
 253 /**
 254  * We've finished publishing the SBlock as part of a larger upload.
 255  * Check the result and complete the larger upload.
 256  *
 257  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 258  * @param uri URI of the published SBlock
 259  * @param emsg NULL on success, otherwise error message
 260  */
 261 static void
 262 publish_sblocks_cont (void *cls,
 263                       const struct GNUNET_FS_Uri *uri,
 264                       const char *emsg)
 265 {
 266   struct GNUNET_FS_PublishContext *pc = cls;
 267   if (NULL != emsg)
 268     {
 269       signal_publish_error (pc->fi,
 270                             pc,
 271                             emsg);
 272       GNUNET_FS_publish_sync_ (pc);
 273       return;
 274     }
 275   GNUNET_assert (pc->qre == NULL);
 276   if ( (pc->dsh != NULL) &&
 277        (pc->rid != 0) )
 278     {
 279       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 280                                                   pc->rid,
 281                                                   UINT_MAX,
 282                                                   UNIT_MAX,
 283                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 284                                                   &finish_release_reserve,
 285                                                   pc);
 286     }
 287   else
 288     {
 289       finish_release_reserve (pc, GNUNET_OK, NULL);
 290     }
 291 }
 292
 293
 294 /**
 295  * We are almost done publishing the structure,
 296  * add SBlocks (if needed).
 297  *
 298  * @param sc overall upload data
 299  */
 300 static void
 301 publish_sblock (struct GNUNET_FS_PublishContext *sc)
 302 {
 303   if (NULL != sc->namespace)
 304     GNUNET_FS_publish_sks (sc->h,
 305                            sc->namespace,
 306                            sc->nid,
 307                            sc->nuid,
 308                            sc->fi->meta,
 309                            sc->fi->chk_uri,
 310                            sc->fi->expirationTime,
 311                            sc->fi->anonymity,
 312                            sc->fi->priority,
 313                            sc->options,
 314                            &publish_sblocks_cont,
 315                            sc);
 316   else
 317     publish_sblocks_cont (sc, NULL, NULL);
 318 }
 319
 320
 321 /**
 322  * We've finished publishing a KBlock as part of a larger upload.
 323  * Check the result and continue the larger upload.
 324  *
 325  * @param cls the "struct GNUNET_FS_PublishContext*"
 326  *        of the larger upload
 327  * @param uri URI of the published blocks
 328  * @param emsg NULL on success, otherwise error message
 329  */
 330 static void
 331 publish_kblocks_cont (void *cls,
 332                       const struct GNUNET_FS_Uri *uri,
 333                       const char *emsg)
 334 {
 335   struct GNUNET_FS_PublishContext *pc = cls;
 336   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 337
 338   if (NULL != emsg)
 339     {
 340 #if DEBUG_PUBLISH
 341       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 342                   "Error uploading KSK blocks: %s\n",
 343                   emsg);
 344 #endif
 345       signal_publish_error (p, pc, emsg);
 346       GNUNET_FS_file_information_sync_ (p);
 347       GNUNET_FS_publish_sync_ (pc);
 348       pc->upload_task
 349         = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 350                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 351                                               &GNUNET_FS_publish_main_,
 352                                               pc);
 353       return;
 354     }
 355 #if DEBUG_PUBLISH
 356   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 357               "KSK blocks published, moving on to next file\n");
 358 #endif
 359   if (NULL != p->dir)
 360     signal_publish_completion (p, pc);
 361   /* move on to next file */
 362   if (NULL != p->next)
 363     pc->fi_pos = p->next;
 364   else
 365     pc->fi_pos = p->dir;
 366   GNUNET_FS_publish_sync_ (pc);
 367   pc->upload_task
 368     = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 369                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 370                                           &GNUNET_FS_publish_main_,
 371                                           pc);
 372 }
 373
 374
 375 /**
 376  * Function called by the tree encoder to obtain
 377  * a block of plaintext data (for the lowest level
 378  * of the tree).
 379  *
 380  * @param cls our publishing context
 381  * @param offset identifies which block to get
 382  * @param max (maximum) number of bytes to get; returning
 383  *        fewer will also cause errors
 384  * @param buf where to copy the plaintext buffer
 385  * @param emsg location to store an error message (on error)
 386  * @return number of bytes copied to buf, 0 on error
 387  */
 388 static size_t
 389 block_reader (void *cls,
 390               uint64_t offset,
 391               size_t max,
 392               void *buf,
 393               char **emsg)
 394 {
 395   struct GNUNET_FS_PublishContext *sc = cls;
 396   struct GNUNET_FS_FileInformation *p;
 397   size_t pt_size;
 398   const char *dd;
 399
 400   p = sc->fi_pos;
 401   if (p->is_directory)
 402     {
 403       pt_size = GNUNET_MIN(max,
 404                            p->data.dir.dir_size - offset);
 405       dd = p->data.dir.dir_data;
 406       memcpy (buf,
 407               &dd[offset],
 408               pt_size);
 409     }
 410   else
 411     {
 412       pt_size = GNUNET_MIN(max,
 413                            p->data.file.file_size - offset);
 414       if (pt_size == 0)
 415         return 0; /* calling reader with pt_size==0
 416                      might free buf, so don't! */
 417       if (pt_size !=
 418           p->data.file.reader (p->data.file.reader_cls,
 419                                offset,
 420                                pt_size,
 421                                buf,
 422                                emsg))
 423         return 0;
 424     }
 425   return pt_size;
 426 }
 427
 428
 429 /**
 430  * The tree encoder has finished processing a
 431  * file.   Call it's finish method and deal with
 432  * the final result.
 433  *
 434  * @param cls our publishing context
 435  * @param tc scheduler's task context (not used)
 436  */
 437 static void
 438 encode_cont (void *cls,
 439              const struct GNUNET_SCHEDULER_TaskContext *tc)
 440 {
 441   struct GNUNET_FS_PublishContext *sc = cls;
 442   struct GNUNET_FS_FileInformation *p;
 443   struct GNUNET_FS_ProgressInfo pi;
 444   char *emsg;
 445
 446   p = sc->fi_pos;
 447   GNUNET_FS_tree_encoder_finish (p->te,
 448                                  &p->chk_uri,
 449                                  &emsg);
 450   p->te = NULL;
 451   if (NULL != emsg)
 452     {
 453 #if DEBUG_PUBLISH
 454       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 455                   "Error during tree walk: %s\n",
 456                   emsg);
 457 #endif
 458       GNUNET_asprintf (&p->emsg,
 459                        _("Publishing failed: %s"),
 460                        emsg);
 461       GNUNET_free (emsg);
 462       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 463       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 464       pi.value.publish.specifics.error.message = p->emsg;
 465       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 466     }
 467 #if DEBUG_PUBLISH
 468   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 469               "Finished with tree encoder\n");
 470 #endif
 471   /* continue with main */
 472   sc->upload_task
 473     = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 474                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 475                                           &GNUNET_FS_publish_main_,
 476                                           sc);
 477 }
 478
 479
 480 /**
 481  * Function called asking for the current (encoded)
 482  * block to be processed.  After processing the
 483  * client should either call "GNUNET_FS_tree_encode_next"
 484  * or (on error) "GNUNET_FS_tree_encode_finish".
 485  *
 486  * @param cls closure
 487  * @param query the query for the block (key for lookup in the datastore)
 488  * @param offset offset of the block in the file
 489  * @param type type of the block (IBLOCK or DBLOCK)
 490  * @param block the (encrypted) block
 491  * @param block_size size of block (in bytes)
 492  */
 493 static void
 494 block_proc (void *cls,
 495             const GNUNET_HashCode *query,
 496             uint64_t offset,
 497             enum GNUNET_BLOCK_Type type,
 498             const void *block,
 499             uint16_t block_size)
 500 {
 501   struct GNUNET_FS_PublishContext *sc = cls;
 502   struct GNUNET_FS_FileInformation *p;
 503   struct PutContCtx * dpc_cls;
 504   struct OnDemandBlock odb;
 505
 506   p = sc->fi_pos;
 507   if (NULL == sc->dsh)
 508     {
 509 #if DEBUG_PUBLISH
 510       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 511                   "Waiting for datastore connection\n");
 512 #endif
 513       sc->upload_task
 514         = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 515                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 516                                               &GNUNET_FS_publish_main_,
 517                                               sc);
 518       return;
 519     }
 520
 521   GNUNET_assert (GNUNET_NO == sc->in_network_wait);
 522   sc->in_network_wait = GNUNET_YES;
 523   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 524   dpc_cls->cont = &GNUNET_FS_publish_main_;
 525   dpc_cls->cont_cls = sc;
 526   dpc_cls->sc = sc;
 527   dpc_cls->p = p;
 528   if ( (! p->is_directory) &&
 529        (GNUNET_YES == p->data.file.do_index) &&
 530        (type == GNUNET_BLOCK_TYPE_DBLOCK) )
 531     {
 532 #if DEBUG_PUBLISH
 533       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 534                   "Indexing block `%s' for offset %llu with index size %u\n",
 535                   GNUNET_h2s (query),
 536                   (unsigned long long) offset,
 537                   sizeof (struct OnDemandBlock));
 538 #endif
 539       odb.offset = GNUNET_htonll (offset);
 540       odb.file_id = p->data.file.file_id;
 541       GNUNET_DATASTORE_put (sc->dsh,
 542                             sc->rid,
 543                             query,
 544                             sizeof(struct OnDemandBlock),
 545                             &odb,
 546                             GNUNET_BLOCK_TYPE_ONDEMAND,
 547                             p->priority,
 548                             p->anonymity,
 549                             p->expirationTime,
 550                             -2, 1,
 551                             GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 552                             &ds_put_cont,
 553                             dpc_cls);
 554       return;
 555     }
 556 #if DEBUG_PUBLISH
 557   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 558               "Publishing block `%s' for offset %llu with size %u\n",
 559               GNUNET_h2s (query),
 560               (unsigned long long) offset,
 561               (unsigned int) block_size);
 562 #endif
 563   GNUNET_DATASTORE_put (sc->dsh,
 564                         sc->rid,
 565                         query,
 566                         block_size,
 567                         block,
 568                         type,
 569                         p->priority,
 570                         p->anonymity,
 571                         p->expirationTime,
 572                         -2, 1,
 573                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 574                         &ds_put_cont,
 575                         dpc_cls);
 576 }
 577
 578
 579 /**
 580  * Function called with information about our
 581  * progress in computing the tree encoding.
 582  *
 583  * @param cls closure
 584  * @param offset where are we in the file
 585  * @param pt_block plaintext of the currently processed block
 586  * @param pt_size size of pt_block
 587  * @param depth depth of the block in the tree
 588  */
 589 static void
 590 progress_proc (void *cls,
 591                uint64_t offset,
 592                const void *pt_block,
 593                size_t pt_size,
 594                unsigned int depth)
 595 {
 596   struct GNUNET_FS_PublishContext *sc = cls;
 597   struct GNUNET_FS_FileInformation *p;
 598   struct GNUNET_FS_ProgressInfo pi;
 599
 600   p = sc->fi_pos;
 601   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 602   pi.value.publish.specifics.progress.data = pt_block;
 603   pi.value.publish.specifics.progress.offset = offset;
 604   pi.value.publish.specifics.progress.data_len = pt_size;
 605   pi.value.publish.specifics.progress.depth = depth;
 606   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset);
 607 }
 608
 609
 610 /**
 611  * We are uploading a file or directory; load (if necessary) the next
 612  * block into memory, encrypt it and send it to the FS service.  Then
 613  * continue with the main task.
 614  *
 615  * @param sc overall upload data
 616  */
 617 static void
 618 publish_content (struct GNUNET_FS_PublishContext *sc)
 619 {
 620   struct GNUNET_FS_FileInformation *p;
 621   char *emsg;
 622   struct GNUNET_FS_DirectoryBuilder *db;
 623   struct GNUNET_FS_FileInformation *dirpos;
 624   void *raw_data;
 625   uint64_t size;
 626
 627   p = sc->fi_pos;
 628   GNUNET_assert (p != NULL);
 629   if (NULL == p->te)
 630     {
 631       if (p->is_directory)
 632         {
 633 #if DEBUG_PUBLISH
 634           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 635                       "Creating directory\n");
 636 #endif
 637           db = GNUNET_FS_directory_builder_create (p->meta);
 638           dirpos = p->data.dir.entries;
 639           while (NULL != dirpos)
 640             {
 641               if (dirpos->is_directory)
 642                 {
 643                   raw_data = dirpos->data.dir.dir_data;
 644                   dirpos->data.dir.dir_data = NULL;
 645                 }
 646               else
 647                 {
 648                   raw_data = NULL;
 649                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 650                        (dirpos->data.file.file_size > 0) )
 651                     {
 652                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 653                       emsg = NULL;
 654                       if (dirpos->data.file.file_size !=
 655                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 656                                                     0,
 657                                                     dirpos->data.file.file_size,
 658                                                     raw_data,
 659                                                     &emsg))
 660                         {
 661                           GNUNET_free_non_null (emsg);
 662                           GNUNET_free (raw_data);
 663                           raw_data = NULL;
 664                         }
 665                     }
 666                 }
 667               GNUNET_FS_directory_builder_add (db,
 668                                                dirpos->chk_uri,
 669                                                dirpos->meta,
 670                                                raw_data);
 671               GNUNET_free_non_null (raw_data);
 672               dirpos = dirpos->next;
 673             }
 674           GNUNET_FS_directory_builder_finish (db,
 675                                               &p->data.dir.dir_size,
 676                                               &p->data.dir.dir_data);
 677           GNUNET_FS_file_information_sync_ (p);
 678         }
 679       size = (p->is_directory)
 680         ? p->data.dir.dir_size
 681         : p->data.file.file_size;
 682 #if DEBUG_PUBLISH
 683       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 684                   "Creating tree encoder\n");
 685 #endif
 686       p->te = GNUNET_FS_tree_encoder_create (sc->h,
 687                                              size,
 688                                              sc,
 689                                              &block_reader,
 690                                              &block_proc,
 691                                              &progress_proc,
 692                                              &encode_cont);
 693
 694     }
 695 #if DEBUG_PUBLISH
 696   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 697               "Processing next block from tree\n");
 698 #endif
 699   GNUNET_FS_tree_encoder_next (p->te);
 700 }
 701
 702
 703 /**
 704  * Process the response (or lack thereof) from
 705  * the "fs" service to our 'start index' request.
 706  *
 707  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 708  * @param msg the response we got
 709  */
 710 static void
 711 process_index_start_response (void *cls,
 712                               const struct GNUNET_MessageHeader *msg)
 713 {
 714   struct GNUNET_FS_PublishContext *sc = cls;
 715   struct GNUNET_FS_FileInformation *p;
 716   const char *emsg;
 717   uint16_t msize;
 718
 719   GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO);
 720   sc->client = NULL;
 721   p = sc->fi_pos;
 722   if (msg == NULL)
 723     {
 724       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 725                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 726                   p->filename,
 727                   _("timeout on index-start request to `fs' service"));
 728       p->data.file.do_index = GNUNET_NO;
 729       GNUNET_FS_file_information_sync_ (p);
 730       publish_content (sc);
 731       return;
 732     }
 733   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 734     {
 735       msize = ntohs (msg->size);
 736       emsg = (const char *) &msg[1];
 737       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 738            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 739         emsg = gettext_noop ("unknown error");
 740       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 741                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 742                   p->filename,
 743                   gettext (emsg));
 744       p->data.file.do_index = GNUNET_NO;
 745       GNUNET_FS_file_information_sync_ (p);
 746       publish_content (sc);
 747       return;
 748     }
 749   p->data.file.index_start_confirmed = GNUNET_YES;
 750   /* success! continue with indexing */
 751   GNUNET_FS_file_information_sync_ (p);
 752   publish_content (sc);
 753 }
 754
 755
 756 /**
 757  * Function called once the hash computation over an
 758  * indexed file has completed.
 759  *
 760  * @param cls closure, our publishing context
 761  * @param res resulting hash, NULL on error
 762  */
 763 static void
 764 hash_for_index_cb (void *cls,
 765                    const GNUNET_HashCode *
 766                    res)
 767 {
 768   struct GNUNET_FS_PublishContext *sc = cls;
 769   struct GNUNET_FS_FileInformation *p;
 770   struct IndexStartMessage *ism;
 771   size_t slen;
 772   struct GNUNET_CLIENT_Connection *client;
 773   uint64_t dev;
 774   uint64_t ino;
 775   char *fn;
 776
 777   p = sc->fi_pos;
 778   if (NULL == res)
 779     {
 780       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 781                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 782                   p->filename,
 783                   _("failed to compute hash"));
 784       p->data.file.do_index = GNUNET_NO;
 785       GNUNET_FS_file_information_sync_ (p);
 786       publish_content (sc);
 787       return;
 788     }
 789   if (GNUNET_YES == p->data.file.index_start_confirmed)
 790     {
 791       publish_content (sc);
 792       return;
 793     }
 794   fn = GNUNET_STRINGS_filename_expand (p->filename);
 795   GNUNET_assert (fn != NULL);
 796   slen = strlen (fn) + 1;
 797   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 798     {
 799       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 800                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 801                   fn,
 802                   _("filename too long"));
 803       GNUNET_free (fn);
 804       p->data.file.do_index = GNUNET_NO;
 805       GNUNET_FS_file_information_sync_ (p);
 806       publish_content (sc);
 807       return;
 808     }
 809 #if DEBUG_PUBLISH
 810   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 811               "Hash of indexed file `%s' is `%s'\n",
 812               p->filename,
 813               GNUNET_h2s (res));
 814 #endif
 815   client = GNUNET_CLIENT_connect (sc->h->sched,
 816                                   "fs",
 817                                   sc->h->cfg);
 818   if (NULL == client)
 819     {
 820       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 821                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 822                   p->filename,
 823                   _("could not connect to `fs' service"));
 824       p->data.file.do_index = GNUNET_NO;
 825       publish_content (sc);
 826       GNUNET_free (fn);
 827       return;
 828     }
 829   if (p->data.file.have_hash != GNUNET_YES)
 830     {
 831       p->data.file.file_id = *res;
 832       p->data.file.have_hash = GNUNET_YES;
 833       GNUNET_FS_file_information_sync_ (p);
 834     }
 835   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 836                        slen);
 837   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 838                            slen);
 839   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 840   if (GNUNET_OK ==
 841       GNUNET_DISK_file_get_identifiers (p->filename,
 842                                         &dev,
 843                                         &ino))
 844     {
 845       ism->device = GNUNET_htonll (dev);
 846       ism->inode = GNUNET_htonll(ino);
 847     }
 848 #if DEBUG_PUBLISH
 849   else
 850     {
 851       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 852                   _("Failed to get file identifiers for `%s'\n"),
 853                   p->filename);
 854     }
 855 #endif
 856   ism->file_id = *res;
 857   memcpy (&ism[1],
 858           fn,
 859           slen);
 860   GNUNET_free (fn);
 861   sc->client = client;
 862   GNUNET_break (GNUNET_YES ==
 863                 GNUNET_CLIENT_transmit_and_get_response (client,
 864                                                          &ism->header,
 865                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 866                                                          GNUNET_YES,
 867                                                          &process_index_start_response,
 868                                                          sc));
 869   GNUNET_free (ism);
 870 }
 871
 872
 873 /**
 874  * Main function that performs the upload.
 875  *
 876  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 877  * @param tc task context
 878  */
 879 void
 880 GNUNET_FS_publish_main_ (void *cls,
 881                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 882 {
 883   struct GNUNET_FS_PublishContext *pc = cls;
 884   struct GNUNET_FS_ProgressInfo pi;
 885   struct GNUNET_FS_FileInformation *p;
 886   char *fn;
 887
 888   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 889   p = pc->fi_pos;
 890   if (NULL == p)
 891     {
 892 #if DEBUG_PUBLISH
 893       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 894                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 895 #endif
 896       /* upload of entire hierarchy complete,
 897          publish namespace entries */
 898       GNUNET_FS_publish_sync_ (pc);
 899       publish_sblock (pc);
 900       return;
 901     }
 902   /* find starting position */
 903   while ( (p->is_directory) &&
 904           (NULL != p->data.dir.entries) &&
 905           (NULL == p->emsg) &&
 906           (NULL == p->data.dir.entries->chk_uri) )
 907     {
 908       p = p->data.dir.entries;
 909       pc->fi_pos = p;
 910       GNUNET_FS_publish_sync_ (pc);
 911     }
 912   /* abort on error */
 913   if (NULL != p->emsg)
 914     {
 915 #if DEBUG_PUBLISH
 916       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 917                   "Error uploading: %s\n",
 918                   p->emsg);
 919 #endif
 920       /* error with current file, abort all
 921          related files as well! */
 922       while (NULL != p->dir)
 923         {
 924           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 925                                                        EXTRACTOR_METATYPE_FILENAME);
 926           p = p->dir;
 927           if (fn != NULL)
 928             {
 929               GNUNET_asprintf (&p->emsg,
 930                                _("Recursive upload failed at `%s': %s"),
 931                                fn,
 932                                p->emsg);
 933               GNUNET_free (fn);
 934             }
 935           else
 936             {
 937               GNUNET_asprintf (&p->emsg,
 938                                _("Recursive upload failed: %s"),
 939                                p->emsg);
 940             }
 941           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 942           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 943           pi.value.publish.specifics.error.message = p->emsg;
 944           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 945         }
 946       pc->all_done = GNUNET_YES;
 947       GNUNET_FS_publish_sync_ (pc);
 948       return;
 949     }
 950   /* handle completion */
 951   if (NULL != p->chk_uri)
 952     {
 953 #if DEBUG_PUBLISH
 954       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 955                   "File upload complete, now publishing KSK blocks.\n");
 956 #endif
 957       GNUNET_FS_publish_sync_ (pc);
 958       /* upload of "p" complete, publish KBlocks! */
 959       if (p->keywords != NULL)
 960         {
 961           GNUNET_FS_publish_ksk (pc->h,
 962                                  p->keywords,
 963                                  p->meta,
 964                                  p->chk_uri,
 965                                  p->expirationTime,
 966                                  p->anonymity,
 967                                  p->priority,
 968                                  pc->options,
 969                                  &publish_kblocks_cont,
 970                                  pc);
 971         }
 972       else
 973         {
 974           publish_kblocks_cont (pc,
 975                                 p->chk_uri,
 976                                 NULL);
 977         }
 978       return;
 979     }
 980   if ( (!p->is_directory) &&
 981        (p->data.file.do_index) )
 982     {
 983       if (NULL == p->filename)
 984         {
 985           p->data.file.do_index = GNUNET_NO;
 986           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 987                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 988                       "<no-name>",
 989                       _("needs to be an actual file"));
 990           GNUNET_FS_file_information_sync_ (p);
 991           publish_content (pc);
 992           return;
 993         }
 994       if (p->data.file.have_hash)
 995         {
 996           hash_for_index_cb (pc,
 997                              &p->data.file.file_id);
 998         }
 999       else
1000         {
1001           p->start_time = GNUNET_TIME_absolute_get ();
1002           GNUNET_CRYPTO_hash_file (pc->h->sched,
1003                                    GNUNET_SCHEDULER_PRIORITY_IDLE,
1004                                    p->filename,
1005                                    HASHING_BLOCKSIZE,
1006                                    &hash_for_index_cb,
1007                                    pc);
1008         }
1009       return;
1010     }
1011   publish_content (pc);
1012 }
1013
1014
1015 /**
1016  * Signal the FS's progress function that we are starting
1017  * an upload.
1018  *
1019  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1020  * @param fi the entry in the publish-structure
1021  * @param length length of the file or directory
1022  * @param meta metadata for the file or directory (can be modified)
1023  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1024  * @param anonymity pointer to selected anonymity level (can be modified)
1025  * @param priority pointer to selected priority (can be modified)
1026  * @param do_index should we index?
1027  * @param expirationTime pointer to selected expiration time (can be modified)
1028  * @param client_info pointer to client context set upon creation (can be modified)
1029  * @return GNUNET_OK to continue (always)
1030  */
1031 static int
1032 fip_signal_start(void *cls,
1033                  struct GNUNET_FS_FileInformation *fi,
1034                  uint64_t length,
1035                  struct GNUNET_CONTAINER_MetaData *meta,
1036                  struct GNUNET_FS_Uri **uri,
1037                  uint32_t *anonymity,
1038                  uint32_t *priority,
1039                  int *do_index,
1040                  struct GNUNET_TIME_Absolute *expirationTime,
1041                  void **client_info)
1042 {
1043   struct GNUNET_FS_PublishContext *sc = cls;
1044   struct GNUNET_FS_ProgressInfo pi;
1045   unsigned int kc;
1046   uint64_t left;
1047
1048   if (*do_index)
1049     {
1050       /* space for on-demand blocks */
1051       sc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1052     }
1053   else
1054     {
1055       /* space for DBlocks */
1056       sc->reserve_space += length;
1057     }
1058   /* entries for IBlocks and DBlocks, space for IBlocks */
1059   left = length;
1060   while (1)
1061     {
1062       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1063       sc->reserve_entries += left;
1064       if (left == 1)
1065         break;
1066       left = left * sizeof (struct ContentHashKey);
1067       sc->reserve_space += left;
1068     }
1069   /* entries and space for keywords */
1070   if (NULL != *uri)
1071     {
1072       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1073       sc->reserve_entries += kc;
1074       sc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1075     }
1076   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1077   *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
1078   GNUNET_FS_file_information_sync_ (fi);
1079   return GNUNET_OK;
1080 }
1081
1082
1083 /**
1084  * Signal the FS's progress function that we are suspending
1085  * an upload.
1086  *
1087  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1088  * @param fi the entry in the publish-structure
1089  * @param length length of the file or directory
1090  * @param meta metadata for the file or directory (can be modified)
1091  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1092  * @param anonymity pointer to selected anonymity level (can be modified)
1093  * @param priority pointer to selected priority (can be modified)
1094  * @param do_index should we index?
1095  * @param expirationTime pointer to selected expiration time (can be modified)
1096  * @param client_info pointer to client context set upon creation (can be modified)
1097  * @return GNUNET_OK to continue (always)
1098  */
1099 static int
1100 fip_signal_suspend(void *cls,
1101                    struct GNUNET_FS_FileInformation *fi,
1102                    uint64_t length,
1103                    struct GNUNET_CONTAINER_MetaData *meta,
1104                    struct GNUNET_FS_Uri **uri,
1105                    uint32_t *anonymity,
1106                    uint32_t *priority,
1107                    int *do_index,
1108                    struct GNUNET_TIME_Absolute *expirationTime,
1109                    void **client_info)
1110 {
1111   struct GNUNET_FS_PublishContext*sc = cls;
1112   struct GNUNET_FS_ProgressInfo pi;
1113   uint64_t off;
1114
1115   GNUNET_free_non_null (fi->serialization);
1116   fi->serialization = NULL;
1117   off = (fi->chk_uri == NULL) ? 0 : length;
1118   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1119   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1120   *client_info = NULL;
1121   if (NULL != sc->dsh)
1122     {
1123       GNUNET_DATASTORE_disconnect (sc->dsh, GNUNET_NO);
1124       sc->dsh = NULL;
1125     }
1126   return GNUNET_OK;
1127 }
1128
1129
1130 /**
1131  * Create SUSPEND event for the given publish operation
1132  * and then clean up our state (without stop signal).
1133  *
1134  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1135  */
1136 void
1137 GNUNET_FS_publish_signal_suspend_ (void *cls)
1138 {
1139   struct GNUNET_FS_PublishContext *pc = cls;
1140
1141   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1142     {
1143       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1144       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1145     }
1146   GNUNET_FS_file_information_inspect (pc->fi,
1147                                       &fip_signal_suspend,
1148                                       pc);
1149   GNUNET_FS_end_top (pc->h, pc->top);
1150   publish_cleanup (pc, NULL);
1151 }
1152
1153
1154 /**
1155  * We have gotten a reply for our space reservation request.
1156  * Either fail (insufficient space) or start publishing for good.
1157  *
1158  * @param cls the 'struct GNUNET_FS_PublishContext*'
1159  * @param success positive reservation ID on success
1160  * @param msg error message on error, otherwise NULL
1161  */
1162 static void
1163 finish_reserve (void *cls,
1164                 int success,
1165                 const char *msg)
1166 {
1167   struct GNUNET_FS_PublishContext *pc = cls;
1168   struct GNUNET_FS_ProgressInfo pi;
1169
1170   pc->qre = NULL;
1171   if ( (msg != NULL) ||
1172        (success <= 0) )
1173     {
1174       GNUNET_asprintf (&pc->fi->emsg,
1175                        _("Insufficient space for publishing: %s"),
1176                        msg);
1177       signal_publish_error (pc->fi,
1178                             pc,
1179                             pc->fi->emsg);
1180       return;
1181     }
1182   pc->rid = success;
1183   ret->upload_task
1184     = GNUNET_SCHEDULER_add_with_priority (h->sched,
1185                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1186                                           &GNUNET_FS_publish_main_,
1187                                           ret);
1188 }
1189
1190
1191 /**
1192  * Publish a file or directory.
1193  *
1194  * @param h handle to the file sharing subsystem
1195  * @param fi information about the file or directory structure to publish
1196  * @param namespace namespace to publish the file in, NULL for no namespace
1197  * @param nid identifier to use for the publishd content in the namespace
1198  *        (can be NULL, must be NULL if namespace is NULL)
1199  * @param nuid update-identifier that will be used for future updates
1200  *        (can be NULL, must be NULL if namespace or nid is NULL)
1201  * @param options options for the publication
1202  * @return context that can be used to control the publish operation
1203  */
1204 struct GNUNET_FS_PublishContext *
1205 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1206                          struct GNUNET_FS_FileInformation *fi,
1207                          struct GNUNET_FS_Namespace *namespace,
1208                          const char *nid,
1209                          const char *nuid,
1210                          enum GNUNET_FS_PublishOptions options)
1211 {
1212   struct GNUNET_FS_PublishContext *ret;
1213   struct GNUNET_DATASTORE_Handle *dsh;
1214
1215   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1216     {
1217       dsh = GNUNET_DATASTORE_connect (h->cfg,
1218                                       h->sched);
1219       if (NULL == dsh)
1220         return NULL;
1221     }
1222   else
1223     {
1224       dsh = NULL;
1225     }
1226   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1227   ret->dsh = dsh;
1228   ret->h = h;
1229   ret->fi = fi;
1230   ret->namespace = namespace;
1231   if (namespace != NULL)
1232     {
1233       namespace->rc++;
1234       GNUNET_assert (NULL != nid);
1235       ret->nid = GNUNET_strdup (nid);
1236       if (NULL != nuid)
1237         ret->nuid = GNUNET_strdup (nuid);
1238     }
1239   /* signal start */
1240   GNUNET_FS_file_information_inspect (ret->fi,
1241                                       &fip_signal_start,
1242                                       ret);
1243   ret->fi_pos = ret->fi;
1244   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1245   GNUNET_FS_publish_sync_ (ret);
1246   if (NULL != ret->dsh)
1247     {
1248       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1249                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1250                   (unsigned int) ret->reserve_entries,
1251                   (unsigned long long) ret->reserve_space);
1252       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1253                                            ret->reserve_space,
1254                                            ret->reserve_entries,
1255                                            UINT_MAX,
1256                                            UINT_MAX,
1257                                            GNUNET_TIME_UNIT_FOREVER_REL,
1258                                            &finish_reserve,
1259                                            ret);
1260     }
1261   else
1262     {
1263       ret->upload_task
1264         = GNUNET_SCHEDULER_add_with_priority (h->sched,
1265                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1266                                               &GNUNET_FS_publish_main_,
1267                                               ret);
1268     }
1269   return ret;
1270 }
1271
1272
1273 /**
1274  * Signal the FS's progress function that we are stopping
1275  * an upload.
1276  *
1277  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1278  * @param fi the entry in the publish-structure
1279  * @param length length of the file or directory
1280  * @param meta metadata for the file or directory (can be modified)
1281  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1282  * @param anonymity pointer to selected anonymity level (can be modified)
1283  * @param priority pointer to selected priority (can be modified)
1284  * @param do_index should we index?
1285  * @param expirationTime pointer to selected expiration time (can be modified)
1286  * @param client_info pointer to client context set upon creation (can be modified)
1287  * @return GNUNET_OK to continue (always)
1288  */
1289 static int
1290 fip_signal_stop(void *cls,
1291                 struct GNUNET_FS_FileInformation *fi,
1292                 uint64_t length,
1293                 struct GNUNET_CONTAINER_MetaData *meta,
1294                 struct GNUNET_FS_Uri **uri,
1295                 uint32_t *anonymity,
1296                 uint32_t *priority,
1297                 int *do_index,
1298                 struct GNUNET_TIME_Absolute *expirationTime,
1299                 void **client_info)
1300 {
1301   struct GNUNET_FS_PublishContext*sc = cls;
1302   struct GNUNET_FS_ProgressInfo pi;
1303   uint64_t off;
1304
1305   if (fi->serialization != NULL)
1306     {
1307       GNUNET_FS_remove_sync_file_ (sc->h,
1308                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1309                                    fi->serialization);
1310       GNUNET_free (fi->serialization);
1311       fi->serialization = NULL;
1312     }
1313   off = (fi->chk_uri == NULL) ? 0 : length;
1314   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1315   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1316   *client_info = NULL;
1317   return GNUNET_OK;
1318 }
1319
1320
1321 /**
1322  * Stop an upload.  Will abort incomplete uploads (but
1323  * not remove blocks that have already been publishd) or
1324  * simply clean up the state for completed uploads.
1325  * Must NOT be called from within the event callback!
1326  *
1327  * @param pc context for the upload to stop
1328  */
1329 void
1330 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1331 {
1332   GNUNET_FS_end_top (pc->h, pc->top);
1333   if (NULL != pc->qre)
1334     {
1335       GNUNET_DATASTORE_cancel (pc->qre);
1336       pc->qre = NULL;
1337     }
1338   if (NULL != pc->dsh)
1339     {
1340       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1341       pc->dsh = NULL;
1342     }
1343   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1344     {
1345       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1346       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1347     }
1348   if (pc->serialization != NULL)
1349     {
1350       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1351       GNUNET_free (pc->serialization);
1352       pc->serialization = NULL;
1353     }
1354   GNUNET_FS_file_information_inspect (pc->fi,
1355                                       &fip_signal_stop,
1356                                       pc);
1357   if (GNUNET_YES == pc->in_network_wait)
1358     {
1359       pc->in_network_wait = GNUNET_SYSERR;
1360       return;
1361     }
1362   publish_cleanup (pc, NULL);
1363 }
1364
1365
1366 /**
1367  * Context for the KSK publication.
1368  */
1369 struct PublishKskContext
1370 {
1371
1372   /**
1373    * Keywords to use.
1374    */
1375   struct GNUNET_FS_Uri *ksk_uri;
1376
1377   /**
1378    * Global FS context.
1379    */
1380   struct GNUNET_FS_Handle *h;
1381
1382   /**
1383    * The master block that we are sending
1384    * (in plaintext), has "mdsize+slen" more
1385    * bytes than the struct would suggest.
1386    */
1387   struct KBlock *kb;
1388
1389   /**
1390    * Buffer of the same size as "kb" for
1391    * the encrypted version.
1392    */
1393   struct KBlock *cpy;
1394
1395   /**
1396    * Handle to the datastore, NULL if we are just
1397    * simulating.
1398    */
1399   struct GNUNET_DATASTORE_Handle *dsh;
1400
1401   /**
1402    * Function to call once we're done.
1403    */
1404   GNUNET_FS_PublishContinuation cont;
1405
1406   /**
1407    * Closure for cont.
1408    */
1409   void *cont_cls;
1410
1411   /**
1412    * When should the KBlocks expire?
1413    */
1414   struct GNUNET_TIME_Absolute expirationTime;
1415
1416   /**
1417    * Size of the serialized metadata.
1418    */
1419   ssize_t mdsize;
1420
1421   /**
1422    * Size of the (CHK) URI as a string.
1423    */
1424   size_t slen;
1425
1426   /**
1427    * Keyword that we are currently processing.
1428    */
1429   unsigned int i;
1430
1431   /**
1432    * Anonymity level for the KBlocks.
1433    */
1434   uint32_t anonymity;
1435
1436   /**
1437    * Priority for the KBlocks.
1438    */
1439   uint32_t priority;
1440 };
1441
1442
1443 /**
1444  * Continuation of "GNUNET_FS_publish_ksk" that performs
1445  * the actual publishing operation (iterating over all
1446  * of the keywords).
1447  *
1448  * @param cls closure of type "struct PublishKskContext*"
1449  * @param tc unused
1450  */
1451 static void
1452 publish_ksk_cont (void *cls,
1453                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1454
1455
1456 /**
1457  * Function called by the datastore API with
1458  * the result from the PUT request.
1459  *
1460  * @param cls closure of type "struct PublishKskContext*"
1461  * @param success GNUNET_OK on success
1462  * @param msg error message (or NULL)
1463  */
1464 static void
1465 kb_put_cont (void *cls,
1466              int success,
1467              const char *msg)
1468 {
1469   struct PublishKskContext *pkc = cls;
1470
1471   if (GNUNET_OK != success)
1472     {
1473       if (NULL != pkc->dsh)
1474         {
1475           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1476           pkc->dsh = NULL;
1477         }
1478       GNUNET_free (pkc->cpy);
1479       GNUNET_free (pkc->kb);
1480       pkc->cont (pkc->cont_cls,
1481                  NULL,
1482                  msg);
1483       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1484       GNUNET_free (pkc);
1485       return;
1486     }
1487   GNUNET_SCHEDULER_add_continuation (pkc->h->sched,
1488                                      &publish_ksk_cont,
1489                                      pkc,
1490                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1491 }
1492
1493
1494 /**
1495  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1496  * publishing operation (iterating over all of the keywords).
1497  *
1498  * @param cls closure of type "struct PublishKskContext*"
1499  * @param tc unused
1500  */
1501 static void
1502 publish_ksk_cont (void *cls,
1503                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1504 {
1505   struct PublishKskContext *pkc = cls;
1506   const char *keyword;
1507   GNUNET_HashCode key;
1508   GNUNET_HashCode query;
1509   struct GNUNET_CRYPTO_AesSessionKey skey;
1510   struct GNUNET_CRYPTO_AesInitializationVector iv;
1511   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1512
1513
1514   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1515        (NULL == pkc->dsh) )
1516     {
1517       if (NULL != pkc->dsh)
1518         {
1519           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1520           pkc->dsh = NULL;
1521         }
1522       GNUNET_free (pkc->cpy);
1523       GNUNET_free (pkc->kb);
1524       pkc->cont (pkc->cont_cls,
1525                  pkc->ksk_uri,
1526                  NULL);
1527       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1528       GNUNET_free (pkc);
1529       return;
1530     }
1531   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1532   /* first character of keyword indicates if it is
1533      mandatory or not -- ignore for hashing */
1534   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1535   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1536   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1537                              pkc->slen + pkc->mdsize,
1538                              &skey,
1539                              &iv,
1540                              &pkc->cpy[1]);
1541   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1542   GNUNET_assert (NULL != pk);
1543   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1544   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1545                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1546                       &query);
1547   GNUNET_assert (GNUNET_OK ==
1548                  GNUNET_CRYPTO_rsa_sign (pk,
1549                                          &pkc->cpy->purpose,
1550                                          &pkc->cpy->signature));
1551   GNUNET_CRYPTO_rsa_key_free (pk);
1552   GNUNET_DATASTORE_put (pkc->dsh,
1553                         0,
1554                         &query,
1555                         pkc->mdsize +
1556                         sizeof (struct KBlock) +
1557                         pkc->slen,
1558                         pkc->cpy,
1559                         GNUNET_BLOCK_TYPE_KBLOCK,
1560                         pkc->priority,
1561                         pkc->anonymity,
1562                         pkc->expirationTime,
1563                         -2, 1,
1564                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1565                         &kb_put_cont,
1566                         pkc);
1567 }
1568
1569
1570 /**
1571  * Publish a CHK under various keywords on GNUnet.
1572  *
1573  * @param h handle to the file sharing subsystem
1574  * @param ksk_uri keywords to use
1575  * @param meta metadata to use
1576  * @param uri URI to refer to in the KBlock
1577  * @param expirationTime when the KBlock expires
1578  * @param anonymity anonymity level for the KBlock
1579  * @param priority priority for the KBlock
1580  * @param options publication options
1581  * @param cont continuation
1582  * @param cont_cls closure for cont
1583  */
1584 void
1585 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1586                        const struct GNUNET_FS_Uri *ksk_uri,
1587                        const struct GNUNET_CONTAINER_MetaData *meta,
1588                        const struct GNUNET_FS_Uri *uri,
1589                        struct GNUNET_TIME_Absolute expirationTime,
1590                        uint32_t anonymity,
1591                        uint32_t priority,
1592                        enum GNUNET_FS_PublishOptions options,
1593                        GNUNET_FS_PublishContinuation cont,
1594                        void *cont_cls)
1595 {
1596   struct PublishKskContext *pkc;
1597   char *uris;
1598   size_t size;
1599   char *kbe;
1600   char *sptr;
1601
1602   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1603   pkc->h = h;
1604   pkc->expirationTime = expirationTime;
1605   pkc->anonymity = anonymity;
1606   pkc->priority = priority;
1607   pkc->cont = cont;
1608   pkc->cont_cls = cont_cls;
1609   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1610     {
1611       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg,
1612                                            h->sched);
1613       if (pkc->dsh == NULL)
1614         {
1615           cont (cont_cls, NULL, _("Could not connect to datastore."));
1616           GNUNET_free (pkc);
1617           return;
1618         }
1619     }
1620   if (meta == NULL)
1621     pkc->mdsize = 0;
1622   else
1623     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1624   GNUNET_assert (pkc->mdsize >= 0);
1625   uris = GNUNET_FS_uri_to_string (uri);
1626   pkc->slen = strlen (uris) + 1;
1627   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1628   if (size > MAX_KBLOCK_SIZE)
1629     {
1630       size = MAX_KBLOCK_SIZE;
1631       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1632     }
1633   pkc->kb = GNUNET_malloc (size);
1634   kbe = (char *) &pkc->kb[1];
1635   memcpy (kbe, uris, pkc->slen);
1636   GNUNET_free (uris);
1637   sptr = &kbe[pkc->slen];
1638   if (meta != NULL)
1639     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1640                                                         &sptr,
1641                                                         pkc->mdsize,
1642                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1643   if (pkc->mdsize == -1)
1644     {
1645       GNUNET_break (0);
1646       GNUNET_free (pkc->kb);
1647       if (pkc->dsh != NULL)
1648         {
1649           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1650           pkc->dsh = NULL;
1651         }
1652       cont (cont_cls, NULL, _("Internal error."));
1653       GNUNET_free (pkc);
1654       return;
1655     }
1656   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1657
1658   pkc->cpy = GNUNET_malloc (size);
1659   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1660                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1661                                   pkc->mdsize +
1662                                   pkc->slen);
1663   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1664   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1665   GNUNET_SCHEDULER_add_continuation (h->sched,
1666                                      &publish_ksk_cont,
1667                                      pkc,
1668                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1669 }
1670
1671
1672 /**
1673  * Context for the SKS publication.
1674  */
1675 struct PublishSksContext
1676 {
1677
1678   /**
1679    * Global FS context.
1680    */
1681   struct GNUNET_FS_Uri *uri;
1682
1683   /**
1684    * Handle to the datastore.
1685    */
1686   struct GNUNET_DATASTORE_Handle *dsh;
1687
1688   /**
1689    * Function to call once we're done.
1690    */
1691   GNUNET_FS_PublishContinuation cont;
1692
1693   /**
1694    * Closure for cont.
1695    */
1696   void *cont_cls;
1697
1698 };
1699
1700
1701 /**
1702  * Function called by the datastore API with
1703  * the result from the PUT (SBlock) request.
1704  *
1705  * @param cls closure of type "struct PublishSksContext*"
1706  * @param success GNUNET_OK on success
1707  * @param msg error message (or NULL)
1708  */
1709 static void
1710 sb_put_cont (void *cls,
1711              int success,
1712              const char *msg)
1713 {
1714   struct PublishSksContext *psc = cls;
1715
1716   if (NULL != psc->dsh)
1717     {
1718       GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO);
1719       psc->dsh = NULL;
1720     }
1721   if (GNUNET_OK != success)
1722     psc->cont (psc->cont_cls,
1723                NULL,
1724                msg);
1725   else
1726     psc->cont (psc->cont_cls,
1727                psc->uri,
1728                NULL);
1729   GNUNET_FS_uri_destroy (psc->uri);
1730   GNUNET_free (psc);
1731 }
1732
1733
1734 /**
1735  * Publish an SBlock on GNUnet.
1736  *
1737  * @param h handle to the file sharing subsystem
1738  * @param namespace namespace to publish in
1739  * @param identifier identifier to use
1740  * @param update update identifier to use
1741  * @param meta metadata to use
1742  * @param uri URI to refer to in the SBlock
1743  * @param expirationTime when the SBlock expires
1744  * @param anonymity anonymity level for the SBlock
1745  * @param priority priority for the SBlock
1746  * @param options publication options
1747  * @param cont continuation
1748  * @param cont_cls closure for cont
1749  */
1750 void
1751 GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
1752                        struct GNUNET_FS_Namespace *namespace,
1753                        const char *identifier,
1754                        const char *update,
1755                        const struct GNUNET_CONTAINER_MetaData *meta,
1756                        const struct GNUNET_FS_Uri *uri,
1757                        struct GNUNET_TIME_Absolute expirationTime,
1758                        uint32_t anonymity,
1759                        uint32_t priority,
1760                        enum GNUNET_FS_PublishOptions options,
1761                        GNUNET_FS_PublishContinuation cont,
1762                        void *cont_cls)
1763 {
1764   struct PublishSksContext *psc;
1765   struct GNUNET_CRYPTO_AesSessionKey sk;
1766   struct GNUNET_CRYPTO_AesInitializationVector iv;
1767   struct GNUNET_FS_Uri *sks_uri;
1768   char *uris;
1769   size_t size;
1770   size_t slen;
1771   size_t nidlen;
1772   size_t idlen;
1773   ssize_t mdsize;
1774   struct SBlock *sb;
1775   struct SBlock *sb_enc;
1776   char *dest;
1777   struct GNUNET_CONTAINER_MetaData *mmeta;
1778   GNUNET_HashCode key;         /* hash of thisId = key */
1779   GNUNET_HashCode id;          /* hash of hc = identifier */
1780   GNUNET_HashCode query;       /* id ^ nsid = DB query */
1781
1782   if (NULL == meta)
1783     mmeta = GNUNET_CONTAINER_meta_data_create ();
1784   else
1785     mmeta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1786   uris = GNUNET_FS_uri_to_string (uri);
1787   slen = strlen (uris) + 1;
1788   idlen = strlen (identifier);
1789   if (update == NULL)
1790     update = "";
1791   nidlen = strlen (update) + 1;
1792   mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (mmeta);
1793   size = sizeof (struct SBlock) + slen + nidlen + mdsize;
1794   if (size > MAX_SBLOCK_SIZE)
1795     {
1796       size = MAX_SBLOCK_SIZE;
1797       mdsize = size - (sizeof (struct SBlock) + slen + nidlen);
1798     }
1799   sb = GNUNET_malloc (sizeof (struct SBlock) + size);
1800   dest = (char *) &sb[1];
1801   memcpy (dest, update, nidlen);
1802   dest += nidlen;
1803   memcpy (dest, uris, slen);
1804   GNUNET_free (uris);
1805   dest += slen;
1806   mdsize = GNUNET_CONTAINER_meta_data_serialize (mmeta,
1807                                                  &dest,
1808                                                  mdsize,
1809                                                  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1810   GNUNET_CONTAINER_meta_data_destroy (mmeta);
1811   if (mdsize == -1)
1812     {
1813       GNUNET_break (0);
1814       GNUNET_free (sb);
1815       cont (cont_cls,
1816             NULL,
1817             _("Internal error."));
1818       return;
1819     }
1820   size = sizeof (struct SBlock) + mdsize + slen + nidlen;
1821   sb_enc = GNUNET_malloc (size);
1822   GNUNET_CRYPTO_hash (identifier, idlen, &key);
1823   GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &id);
1824   sks_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri));
1825   sks_uri->type = sks;
1826   GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &sb_enc->subspace);
1827   GNUNET_CRYPTO_hash (&sb_enc->subspace,
1828                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1829                       &sks_uri->data.sks.namespace);
1830   sks_uri->data.sks.identifier = GNUNET_strdup (identifier);
1831   GNUNET_CRYPTO_hash_xor (&id,
1832                           &sks_uri->data.sks.namespace,
1833                           &sb_enc->identifier);
1834   GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
1835   GNUNET_CRYPTO_aes_encrypt (&sb[1],
1836                              size - sizeof (struct SBlock),
1837                              &sk,
1838                              &iv,
1839                              &sb_enc[1]);
1840   sb_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK);
1841   sb_enc->purpose.size = htonl(slen + mdsize + nidlen
1842                                + sizeof(struct SBlock)
1843                                - sizeof(struct GNUNET_CRYPTO_RsaSignature));
1844   GNUNET_assert (GNUNET_OK ==
1845                  GNUNET_CRYPTO_rsa_sign (namespace->key,
1846                                          &sb_enc->purpose,
1847                                          &sb_enc->signature));
1848   psc = GNUNET_malloc (sizeof(struct PublishSksContext));
1849   psc->uri = sks_uri;
1850   psc->cont = cont;
1851   psc->cont_cls = cont_cls;
1852   if (0 != (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1853     {
1854       GNUNET_free (sb_enc);
1855       GNUNET_free (sb);
1856       sb_put_cont (psc,
1857                    GNUNET_OK,
1858                    NULL);
1859       return;
1860     }
1861   psc->dsh = GNUNET_DATASTORE_connect (h->cfg, h->sched);
1862   if (NULL == psc->dsh)
1863     {
1864       GNUNET_free (sb_enc);
1865       GNUNET_free (sb);
1866       sb_put_cont (psc,
1867                    GNUNET_NO,
1868                    _("Failed to connect to datastore."));
1869       return;
1870     }
1871   GNUNET_CRYPTO_hash_xor (&sks_uri->data.sks.namespace,
1872                           &id,
1873                           &query);
1874   GNUNET_DATASTORE_put (psc->dsh,
1875                         0,
1876                         &sb_enc->identifier,
1877                         size,
1878                         sb_enc,
1879                         GNUNET_BLOCK_TYPE_SBLOCK,
1880                         priority,
1881                         anonymity,
1882                         expirationTime,
1883                         -2, 1,
1884                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1885                         &sb_put_cont,
1886                         psc);
1887   GNUNET_free (sb);
1888   GNUNET_free (sb_enc);
1889 }
1890
1891 /* end of fs_publish.c */