src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see http://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Context for "ds_put_cont".
  42  */
  43 struct PutContCtx
  44 {
  45   /**
  46    * Current publishing context.
  47    */
  48   struct GNUNET_FS_PublishContext *sc;
  49
  50   /**
  51    * Specific file with the block.
  52    */
  53   struct GNUNET_FS_FileInformation *p;
  54
  55   /**
  56    * Function to run next, if any (can be NULL).
  57    */
  58   GNUNET_SCHEDULER_Task cont;
  59
  60   /**
  61    * Closure for cont.
  62    */
  63   void *cont_cls;
  64 };
  65
  66
  67 /**
  68  * Fill in all of the generic fields for
  69  * a publish event and call the callback.
  70  *
  71  * @param pi structure to fill in
  72  * @param sc overall publishing context
  73  * @param p file information for the file being published
  74  * @param offset where in the file are we so far
  75  * @return value returned from callback
  76  */
  77 void *
  78 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  79                                 struct GNUNET_FS_PublishContext *sc,
  80                                 const struct GNUNET_FS_FileInformation *p,
  81                                 uint64_t offset)
  82 {
  83   pi->value.publish.pc = sc;
  84   pi->value.publish.fi = p;
  85   pi->value.publish.cctx
  86     = p->client_info;
  87   pi->value.publish.pctx
  88     = (NULL == p->dir) ? NULL : p->dir->client_info;
  89   pi->value.publish.filename = p->filename;
  90   pi->value.publish.size
  91     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  92   pi->value.publish.eta
  93     = GNUNET_TIME_calculate_eta (p->start_time,
  94                                  offset,
  95                                  pi->value.publish.size);
  96   pi->value.publish.completed = offset;
  97   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  98   pi->value.publish.anonymity = p->anonymity;
  99   return sc->h->upcb (sc->h->upcb_cls,
 100                       pi);
 101 }
 102
 103
 104 /**
 105  * Cleanup the publish context, we're done with it.
 106  *
 107  * @param cls struct to clean up after
 108  * @param tc scheduler context
 109  */
 110 static void
 111 publish_cleanup (void *cls,
 112                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 113 {
 114   struct GNUNET_FS_PublishContext *pc = cls;
 115
 116   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 117   if (pc->namespace != NULL)
 118     GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 119   GNUNET_free_non_null (pc->nid);
 120   GNUNET_free_non_null (pc->nuid);
 121   GNUNET_free_non_null (pc->serialization);
 122   if (pc->dsh != NULL)
 123     {
 124       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 125       pc->dsh = NULL;
 126     }
 127   if (pc->client != NULL)
 128     GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 129   GNUNET_free (pc);
 130 }
 131
 132
 133 /**
 134  * Function called by the datastore API with
 135  * the result from the PUT request.
 136  *
 137  * @param cls our closure
 138  * @param success GNUNET_OK on success
 139  * @param msg error message (or NULL)
 140  */
 141 static void
 142 ds_put_cont (void *cls,
 143              int success,
 144              const char *msg)
 145 {
 146   struct PutContCtx *pcc = cls;
 147   struct GNUNET_FS_ProgressInfo pi;
 148
 149   if (GNUNET_SYSERR == pcc->sc->in_network_wait)
 150     {
 151       /* we were aborted in the meantime, finish shutdown! */
 152       GNUNET_SCHEDULER_add_continuation (pcc->sc->h->sched,
 153                                          &publish_cleanup,
 154                                          pcc->sc,
 155                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 156       GNUNET_free (pcc);
 157       return;
 158     }
 159   GNUNET_assert (GNUNET_YES == pcc->sc->in_network_wait);
 160   pcc->sc->in_network_wait = GNUNET_NO;
 161   if (GNUNET_OK != success)
 162     {
 163       GNUNET_asprintf (&pcc->p->emsg,
 164                        _("Publishing failed: %s"),
 165                        msg);
 166       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 167       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 168       pi.value.publish.specifics.error.message = pcc->p->emsg;
 169       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->sc, pcc->p, 0);
 170       if ( (pcc->p->is_directory == GNUNET_NO) &&
 171            (pcc->p->filename != NULL) &&
 172            (pcc->p->data.do_index == GNUNET_YES) )
 173         {
 174           /* run unindex to clean up */
 175           GNUNET_FS_unindex_start (pcc->sc->h,
 176                                    pcc->p->filename,
 177                                    NULL);
 178         }
 179     }
 180   if (NULL != pcc->cont)
 181     pcc->sc->upload_task
 182       = GNUNET_SCHEDULER_add_with_priority (pcc->sc->h->sched,
 183                                             GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 184                                             pcc->cont,
 185                                             pcc->cont_cls);
 186   GNUNET_free (pcc);
 187 }
 188
 189
 190 /**
 191  * Generate the callback that signals clients
 192  * that a file (or directory) has been completely
 193  * published.
 194  *
 195  * @param p the completed upload
 196  * @param sc context of the publication
 197  */
 198 static void
 199 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 200                            struct GNUNET_FS_PublishContext *sc)
 201 {
 202   struct GNUNET_FS_ProgressInfo pi;
 203
 204   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 205   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 206   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 207   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p,
 208                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 209 }
 210
 211
 212 /**
 213  * Generate the callback that signals clients
 214  * that a file (or directory) has encountered
 215  * a problem during publication.
 216  *
 217  * @param p the upload that had trouble
 218  * @param sc context of the publication
 219  * @param emsg error message
 220  */
 221 static void
 222 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 223                       struct GNUNET_FS_PublishContext *sc,
 224                       const char *emsg)
 225 {
 226   struct GNUNET_FS_ProgressInfo pi;
 227
 228   p->emsg = GNUNET_strdup (emsg);
 229   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 230   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 231   pi.value.publish.specifics.error.message =emsg;
 232   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 233   if ( (p->is_directory == GNUNET_NO) &&
 234        (p->filename != NULL) &&
 235        (p->data.do_index == GNUNET_YES) )
 236     {
 237       /* run unindex to clean up */
 238       GNUNET_FS_unindex_start (pc->h,
 239                                p->filename,
 240                                NULL);
 241     }
 242
 243 }
 244
 245
 246 /**
 247  * Datastore returns from reservation cancel request.
 248  *
 249  * @param cls the 'struct GNUNET_FS_PublishContext'
 250  * @param success success code (not used)
 251  * @param msg error message (typically NULL, not used)
 252  */
 253 static void
 254 finish_release_reserve (void *cls,
 255                         int success,
 256                         const char *msg)
 257 {
 258   struct GNUNET_FS_PublishContext *pc = cls;
 259
 260   pc->qre = NULL;
 261   signal_publish_completion (pc->fi, pc);
 262   pc->all_done = GNUNET_YES;
 263   GNUNET_FS_publish_sync_ (pc);
 264 }
 265
 266
 267 /**
 268  * We've finished publishing the SBlock as part of a larger upload.
 269  * Check the result and complete the larger upload.
 270  *
 271  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 272  * @param uri URI of the published SBlock
 273  * @param emsg NULL on success, otherwise error message
 274  */
 275 static void
 276 publish_sblocks_cont (void *cls,
 277                       const struct GNUNET_FS_Uri *uri,
 278                       const char *emsg)
 279 {
 280   struct GNUNET_FS_PublishContext *pc = cls;
 281   if (NULL != emsg)
 282     {
 283       signal_publish_error (pc->fi,
 284                             pc,
 285                             emsg);
 286       GNUNET_FS_publish_sync_ (pc);
 287       return;
 288     }
 289   GNUNET_assert (pc->qre == NULL);
 290   if ( (pc->dsh != NULL) &&
 291        (pc->rid != 0) )
 292     {
 293       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 294                                                   pc->rid,
 295                                                   UINT_MAX,
 296                                                   UINT_MAX,
 297                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 298                                                   &finish_release_reserve,
 299                                                   pc);
 300     }
 301   else
 302     {
 303       finish_release_reserve (pc, GNUNET_OK, NULL);
 304     }
 305 }
 306
 307
 308 /**
 309  * We are almost done publishing the structure,
 310  * add SBlocks (if needed).
 311  *
 312  * @param sc overall upload data
 313  */
 314 static void
 315 publish_sblock (struct GNUNET_FS_PublishContext *sc)
 316 {
 317   if (NULL != sc->namespace)
 318     GNUNET_FS_publish_sks (sc->h,
 319                            sc->namespace,
 320                            sc->nid,
 321                            sc->nuid,
 322                            sc->fi->meta,
 323                            sc->fi->chk_uri,
 324                            sc->fi->expirationTime,
 325                            sc->fi->anonymity,
 326                            sc->fi->priority,
 327                            sc->options,
 328                            &publish_sblocks_cont,
 329                            sc);
 330   else
 331     publish_sblocks_cont (sc, NULL, NULL);
 332 }
 333
 334
 335 /**
 336  * We've finished publishing a KBlock as part of a larger upload.
 337  * Check the result and continue the larger upload.
 338  *
 339  * @param cls the "struct GNUNET_FS_PublishContext*"
 340  *        of the larger upload
 341  * @param uri URI of the published blocks
 342  * @param emsg NULL on success, otherwise error message
 343  */
 344 static void
 345 publish_kblocks_cont (void *cls,
 346                       const struct GNUNET_FS_Uri *uri,
 347                       const char *emsg)
 348 {
 349   struct GNUNET_FS_PublishContext *pc = cls;
 350   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 351
 352   if (NULL != emsg)
 353     {
 354 #if DEBUG_PUBLISH
 355       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 356                   "Error uploading KSK blocks: %s\n",
 357                   emsg);
 358 #endif
 359       signal_publish_error (p, pc, emsg);
 360       GNUNET_FS_file_information_sync_ (p);
 361       GNUNET_FS_publish_sync_ (pc);
 362       pc->upload_task
 363         = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 364                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 365                                               &GNUNET_FS_publish_main_,
 366                                               pc);
 367       return;
 368     }
 369 #if DEBUG_PUBLISH
 370   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 371               "KSK blocks published, moving on to next file\n");
 372 #endif
 373   if (NULL != p->dir)
 374     signal_publish_completion (p, pc);
 375   /* move on to next file */
 376   if (NULL != p->next)
 377     pc->fi_pos = p->next;
 378   else
 379     pc->fi_pos = p->dir;
 380   GNUNET_FS_publish_sync_ (pc);
 381   pc->upload_task
 382     = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
 383                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 384                                           &GNUNET_FS_publish_main_,
 385                                           pc);
 386 }
 387
 388
 389 /**
 390  * Function called by the tree encoder to obtain
 391  * a block of plaintext data (for the lowest level
 392  * of the tree).
 393  *
 394  * @param cls our publishing context
 395  * @param offset identifies which block to get
 396  * @param max (maximum) number of bytes to get; returning
 397  *        fewer will also cause errors
 398  * @param buf where to copy the plaintext buffer
 399  * @param emsg location to store an error message (on error)
 400  * @return number of bytes copied to buf, 0 on error
 401  */
 402 static size_t
 403 block_reader (void *cls,
 404               uint64_t offset,
 405               size_t max,
 406               void *buf,
 407               char **emsg)
 408 {
 409   struct GNUNET_FS_PublishContext *sc = cls;
 410   struct GNUNET_FS_FileInformation *p;
 411   size_t pt_size;
 412   const char *dd;
 413
 414   p = sc->fi_pos;
 415   if (p->is_directory)
 416     {
 417       pt_size = GNUNET_MIN(max,
 418                            p->data.dir.dir_size - offset);
 419       dd = p->data.dir.dir_data;
 420       memcpy (buf,
 421               &dd[offset],
 422               pt_size);
 423     }
 424   else
 425     {
 426       pt_size = GNUNET_MIN(max,
 427                            p->data.file.file_size - offset);
 428       if (pt_size == 0)
 429         return 0; /* calling reader with pt_size==0
 430                      might free buf, so don't! */
 431       if (pt_size !=
 432           p->data.file.reader (p->data.file.reader_cls,
 433                                offset,
 434                                pt_size,
 435                                buf,
 436                                emsg))
 437         return 0;
 438     }
 439   return pt_size;
 440 }
 441
 442
 443 /**
 444  * The tree encoder has finished processing a
 445  * file.   Call it's finish method and deal with
 446  * the final result.
 447  *
 448  * @param cls our publishing context
 449  * @param tc scheduler's task context (not used)
 450  */
 451 static void
 452 encode_cont (void *cls,
 453              const struct GNUNET_SCHEDULER_TaskContext *tc)
 454 {
 455   struct GNUNET_FS_PublishContext *sc = cls;
 456   struct GNUNET_FS_FileInformation *p;
 457   struct GNUNET_FS_ProgressInfo pi;
 458   char *emsg;
 459
 460   p = sc->fi_pos;
 461   GNUNET_FS_tree_encoder_finish (p->te,
 462                                  &p->chk_uri,
 463                                  &emsg);
 464   p->te = NULL;
 465   if (NULL != emsg)
 466     {
 467 #if DEBUG_PUBLISH
 468       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 469                   "Error during tree walk: %s\n",
 470                   emsg);
 471 #endif
 472       GNUNET_asprintf (&p->emsg,
 473                        _("Publishing failed: %s"),
 474                        emsg);
 475       GNUNET_free (emsg);
 476       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 477       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 478       pi.value.publish.specifics.error.message = p->emsg;
 479       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, sc, p, 0);
 480     }
 481 #if DEBUG_PUBLISH
 482   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 483               "Finished with tree encoder\n");
 484 #endif
 485   /* continue with main */
 486   sc->upload_task
 487     = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 488                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 489                                           &GNUNET_FS_publish_main_,
 490                                           sc);
 491 }
 492
 493
 494 /**
 495  * Function called asking for the current (encoded)
 496  * block to be processed.  After processing the
 497  * client should either call "GNUNET_FS_tree_encode_next"
 498  * or (on error) "GNUNET_FS_tree_encode_finish".
 499  *
 500  * @param cls closure
 501  * @param query the query for the block (key for lookup in the datastore)
 502  * @param offset offset of the block in the file
 503  * @param type type of the block (IBLOCK or DBLOCK)
 504  * @param block the (encrypted) block
 505  * @param block_size size of block (in bytes)
 506  */
 507 static void
 508 block_proc (void *cls,
 509             const GNUNET_HashCode *query,
 510             uint64_t offset,
 511             enum GNUNET_BLOCK_Type type,
 512             const void *block,
 513             uint16_t block_size)
 514 {
 515   struct GNUNET_FS_PublishContext *sc = cls;
 516   struct GNUNET_FS_FileInformation *p;
 517   struct PutContCtx * dpc_cls;
 518   struct OnDemandBlock odb;
 519
 520   p = sc->fi_pos;
 521   if (NULL == sc->dsh)
 522     {
 523 #if DEBUG_PUBLISH
 524       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 525                   "Waiting for datastore connection\n");
 526 #endif
 527       sc->upload_task
 528         = GNUNET_SCHEDULER_add_with_priority (sc->h->sched,
 529                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 530                                               &GNUNET_FS_publish_main_,
 531                                               sc);
 532       return;
 533     }
 534
 535   GNUNET_assert (GNUNET_NO == sc->in_network_wait);
 536   sc->in_network_wait = GNUNET_YES;
 537   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 538   dpc_cls->cont = &GNUNET_FS_publish_main_;
 539   dpc_cls->cont_cls = sc;
 540   dpc_cls->sc = sc;
 541   dpc_cls->p = p;
 542   if ( (! p->is_directory) &&
 543        (GNUNET_YES == p->data.file.do_index) &&
 544        (type == GNUNET_BLOCK_TYPE_DBLOCK) )
 545     {
 546 #if DEBUG_PUBLISH
 547       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 548                   "Indexing block `%s' for offset %llu with index size %u\n",
 549                   GNUNET_h2s (query),
 550                   (unsigned long long) offset,
 551                   sizeof (struct OnDemandBlock));
 552 #endif
 553       odb.offset = GNUNET_htonll (offset);
 554       odb.file_id = p->data.file.file_id;
 555       GNUNET_DATASTORE_put (sc->dsh,
 556                             sc->rid,
 557                             query,
 558                             sizeof(struct OnDemandBlock),
 559                             &odb,
 560                             GNUNET_BLOCK_TYPE_ONDEMAND,
 561                             p->priority,
 562                             p->anonymity,
 563                             p->expirationTime,
 564                             -2, 1,
 565                             GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 566                             &ds_put_cont,
 567                             dpc_cls);
 568       return;
 569     }
 570 #if DEBUG_PUBLISH
 571   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 572               "Publishing block `%s' for offset %llu with size %u\n",
 573               GNUNET_h2s (query),
 574               (unsigned long long) offset,
 575               (unsigned int) block_size);
 576 #endif
 577   GNUNET_DATASTORE_put (sc->dsh,
 578                         sc->rid,
 579                         query,
 580                         block_size,
 581                         block,
 582                         type,
 583                         p->priority,
 584                         p->anonymity,
 585                         p->expirationTime,
 586                         -2, 1,
 587                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 588                         &ds_put_cont,
 589                         dpc_cls);
 590 }
 591
 592
 593 /**
 594  * Function called with information about our
 595  * progress in computing the tree encoding.
 596  *
 597  * @param cls closure
 598  * @param offset where are we in the file
 599  * @param pt_block plaintext of the currently processed block
 600  * @param pt_size size of pt_block
 601  * @param depth depth of the block in the tree
 602  */
 603 static void
 604 progress_proc (void *cls,
 605                uint64_t offset,
 606                const void *pt_block,
 607                size_t pt_size,
 608                unsigned int depth)
 609 {
 610   struct GNUNET_FS_PublishContext *sc = cls;
 611   struct GNUNET_FS_FileInformation *p;
 612   struct GNUNET_FS_ProgressInfo pi;
 613
 614   p = sc->fi_pos;
 615   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 616   pi.value.publish.specifics.progress.data = pt_block;
 617   pi.value.publish.specifics.progress.offset = offset;
 618   pi.value.publish.specifics.progress.data_len = pt_size;
 619   pi.value.publish.specifics.progress.depth = depth;
 620   p->client_info = GNUNET_FS_publish_make_status_ (&pi, sc, p, offset);
 621 }
 622
 623
 624 /**
 625  * We are uploading a file or directory; load (if necessary) the next
 626  * block into memory, encrypt it and send it to the FS service.  Then
 627  * continue with the main task.
 628  *
 629  * @param sc overall upload data
 630  */
 631 static void
 632 publish_content (struct GNUNET_FS_PublishContext *sc)
 633 {
 634   struct GNUNET_FS_FileInformation *p;
 635   char *emsg;
 636   struct GNUNET_FS_DirectoryBuilder *db;
 637   struct GNUNET_FS_FileInformation *dirpos;
 638   void *raw_data;
 639   uint64_t size;
 640
 641   p = sc->fi_pos;
 642   GNUNET_assert (p != NULL);
 643   if (NULL == p->te)
 644     {
 645       if (p->is_directory)
 646         {
 647 #if DEBUG_PUBLISH
 648           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 649                       "Creating directory\n");
 650 #endif
 651           db = GNUNET_FS_directory_builder_create (p->meta);
 652           dirpos = p->data.dir.entries;
 653           while (NULL != dirpos)
 654             {
 655               if (dirpos->is_directory)
 656                 {
 657                   raw_data = dirpos->data.dir.dir_data;
 658                   dirpos->data.dir.dir_data = NULL;
 659                 }
 660               else
 661                 {
 662                   raw_data = NULL;
 663                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 664                        (dirpos->data.file.file_size > 0) )
 665                     {
 666                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 667                       emsg = NULL;
 668                       if (dirpos->data.file.file_size !=
 669                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 670                                                     0,
 671                                                     dirpos->data.file.file_size,
 672                                                     raw_data,
 673                                                     &emsg))
 674                         {
 675                           GNUNET_free_non_null (emsg);
 676                           GNUNET_free (raw_data);
 677                           raw_data = NULL;
 678                         }
 679                     }
 680                 }
 681               GNUNET_FS_directory_builder_add (db,
 682                                                dirpos->chk_uri,
 683                                                dirpos->meta,
 684                                                raw_data);
 685               GNUNET_free_non_null (raw_data);
 686               dirpos = dirpos->next;
 687             }
 688           GNUNET_FS_directory_builder_finish (db,
 689                                               &p->data.dir.dir_size,
 690                                               &p->data.dir.dir_data);
 691           GNUNET_FS_file_information_sync_ (p);
 692         }
 693       size = (p->is_directory)
 694         ? p->data.dir.dir_size
 695         : p->data.file.file_size;
 696 #if DEBUG_PUBLISH
 697       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 698                   "Creating tree encoder\n");
 699 #endif
 700       p->te = GNUNET_FS_tree_encoder_create (sc->h,
 701                                              size,
 702                                              sc,
 703                                              &block_reader,
 704                                              &block_proc,
 705                                              &progress_proc,
 706                                              &encode_cont);
 707
 708     }
 709 #if DEBUG_PUBLISH
 710   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 711               "Processing next block from tree\n");
 712 #endif
 713   GNUNET_FS_tree_encoder_next (p->te);
 714 }
 715
 716
 717 /**
 718  * Process the response (or lack thereof) from
 719  * the "fs" service to our 'start index' request.
 720  *
 721  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 722  * @param msg the response we got
 723  */
 724 static void
 725 process_index_start_response (void *cls,
 726                               const struct GNUNET_MessageHeader *msg)
 727 {
 728   struct GNUNET_FS_PublishContext *sc = cls;
 729   struct GNUNET_FS_FileInformation *p;
 730   const char *emsg;
 731   uint16_t msize;
 732
 733   GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO);
 734   sc->client = NULL;
 735   p = sc->fi_pos;
 736   if (msg == NULL)
 737     {
 738       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 739                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 740                   p->filename,
 741                   _("timeout on index-start request to `fs' service"));
 742       p->data.file.do_index = GNUNET_NO;
 743       GNUNET_FS_file_information_sync_ (p);
 744       publish_content (sc);
 745       return;
 746     }
 747   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 748     {
 749       msize = ntohs (msg->size);
 750       emsg = (const char *) &msg[1];
 751       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 752            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 753         emsg = gettext_noop ("unknown error");
 754       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 755                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 756                   p->filename,
 757                   gettext (emsg));
 758       p->data.file.do_index = GNUNET_NO;
 759       GNUNET_FS_file_information_sync_ (p);
 760       publish_content (sc);
 761       return;
 762     }
 763   p->data.file.index_start_confirmed = GNUNET_YES;
 764   /* success! continue with indexing */
 765   GNUNET_FS_file_information_sync_ (p);
 766   publish_content (sc);
 767 }
 768
 769
 770 /**
 771  * Function called once the hash computation over an
 772  * indexed file has completed.
 773  *
 774  * @param cls closure, our publishing context
 775  * @param res resulting hash, NULL on error
 776  */
 777 static void
 778 hash_for_index_cb (void *cls,
 779                    const GNUNET_HashCode *
 780                    res)
 781 {
 782   struct GNUNET_FS_PublishContext *sc = cls;
 783   struct GNUNET_FS_FileInformation *p;
 784   struct IndexStartMessage *ism;
 785   size_t slen;
 786   struct GNUNET_CLIENT_Connection *client;
 787   uint64_t dev;
 788   uint64_t ino;
 789   char *fn;
 790
 791   p = sc->fi_pos;
 792   if (NULL == res)
 793     {
 794       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 795                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 796                   p->filename,
 797                   _("failed to compute hash"));
 798       p->data.file.do_index = GNUNET_NO;
 799       GNUNET_FS_file_information_sync_ (p);
 800       publish_content (sc);
 801       return;
 802     }
 803   if (GNUNET_YES == p->data.file.index_start_confirmed)
 804     {
 805       publish_content (sc);
 806       return;
 807     }
 808   fn = GNUNET_STRINGS_filename_expand (p->filename);
 809   GNUNET_assert (fn != NULL);
 810   slen = strlen (fn) + 1;
 811   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 812     {
 813       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 814                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 815                   fn,
 816                   _("filename too long"));
 817       GNUNET_free (fn);
 818       p->data.file.do_index = GNUNET_NO;
 819       GNUNET_FS_file_information_sync_ (p);
 820       publish_content (sc);
 821       return;
 822     }
 823 #if DEBUG_PUBLISH
 824   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 825               "Hash of indexed file `%s' is `%s'\n",
 826               p->filename,
 827               GNUNET_h2s (res));
 828 #endif
 829   client = GNUNET_CLIENT_connect (sc->h->sched,
 830                                   "fs",
 831                                   sc->h->cfg);
 832   if (NULL == client)
 833     {
 834       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 835                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 836                   p->filename,
 837                   _("could not connect to `fs' service"));
 838       p->data.file.do_index = GNUNET_NO;
 839       publish_content (sc);
 840       GNUNET_free (fn);
 841       return;
 842     }
 843   if (p->data.file.have_hash != GNUNET_YES)
 844     {
 845       p->data.file.file_id = *res;
 846       p->data.file.have_hash = GNUNET_YES;
 847       GNUNET_FS_file_information_sync_ (p);
 848     }
 849   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 850                        slen);
 851   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 852                            slen);
 853   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 854   if (GNUNET_OK ==
 855       GNUNET_DISK_file_get_identifiers (p->filename,
 856                                         &dev,
 857                                         &ino))
 858     {
 859       ism->device = GNUNET_htonll (dev);
 860       ism->inode = GNUNET_htonll(ino);
 861     }
 862 #if DEBUG_PUBLISH
 863   else
 864     {
 865       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 866                   _("Failed to get file identifiers for `%s'\n"),
 867                   p->filename);
 868     }
 869 #endif
 870   ism->file_id = *res;
 871   memcpy (&ism[1],
 872           fn,
 873           slen);
 874   GNUNET_free (fn);
 875   sc->client = client;
 876   GNUNET_break (GNUNET_YES ==
 877                 GNUNET_CLIENT_transmit_and_get_response (client,
 878                                                          &ism->header,
 879                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 880                                                          GNUNET_YES,
 881                                                          &process_index_start_response,
 882                                                          sc));
 883   GNUNET_free (ism);
 884 }
 885
 886
 887 /**
 888  * Main function that performs the upload.
 889  *
 890  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 891  * @param tc task context
 892  */
 893 void
 894 GNUNET_FS_publish_main_ (void *cls,
 895                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 896 {
 897   struct GNUNET_FS_PublishContext *pc = cls;
 898   struct GNUNET_FS_ProgressInfo pi;
 899   struct GNUNET_FS_FileInformation *p;
 900   struct GNUNET_FS_Uri *loc;
 901   char *fn;
 902
 903   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 904   p = pc->fi_pos;
 905   if (NULL == p)
 906     {
 907 #if DEBUG_PUBLISH
 908       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 909                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 910 #endif
 911       /* upload of entire hierarchy complete,
 912          publish namespace entries */
 913       GNUNET_FS_publish_sync_ (pc);
 914       publish_sblock (pc);
 915       return;
 916     }
 917   /* find starting position */
 918   while ( (p->is_directory) &&
 919           (NULL != p->data.dir.entries) &&
 920           (NULL == p->emsg) &&
 921           (NULL == p->data.dir.entries->chk_uri) )
 922     {
 923       p = p->data.dir.entries;
 924       pc->fi_pos = p;
 925       GNUNET_FS_publish_sync_ (pc);
 926     }
 927   /* abort on error */
 928   if (NULL != p->emsg)
 929     {
 930 #if DEBUG_PUBLISH
 931       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 932                   "Error uploading: %s\n",
 933                   p->emsg);
 934 #endif
 935       /* error with current file, abort all
 936          related files as well! */
 937       while (NULL != p->dir)
 938         {
 939           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 940                                                        EXTRACTOR_METATYPE_FILENAME);
 941           p = p->dir;
 942           if (fn != NULL)
 943             {
 944               GNUNET_asprintf (&p->emsg,
 945                                _("Recursive upload failed at `%s': %s"),
 946                                fn,
 947                                p->emsg);
 948               GNUNET_free (fn);
 949             }
 950           else
 951             {
 952               GNUNET_asprintf (&p->emsg,
 953                                _("Recursive upload failed: %s"),
 954                                p->emsg);
 955             }
 956           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 957           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 958           pi.value.publish.specifics.error.message = p->emsg;
 959           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 960         }
 961       pc->all_done = GNUNET_YES;
 962       GNUNET_FS_publish_sync_ (pc);
 963       return;
 964     }
 965   /* handle completion */
 966   if (NULL != p->chk_uri)
 967     {
 968 #if DEBUG_PUBLISH
 969       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 970                   "File upload complete, now publishing KSK blocks.\n");
 971 #endif
 972       if (0 == p->anonymity)
 973         {
 974           /* zero anonymity, box CHK URI in LOC URI */
 975           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
 976                                           pc->h->cfg,
 977                                           p->expirationTime);
 978           GNUNET_FS_uri_destroy (p->chk_uri);
 979           p->chk_uri = loc;
 980         }
 981       GNUNET_FS_publish_sync_ (pc);
 982       /* upload of "p" complete, publish KBlocks! */
 983       if (p->keywords != NULL)
 984         {
 985           GNUNET_FS_publish_ksk (pc->h,
 986                                  p->keywords,
 987                                  p->meta,
 988                                  p->chk_uri,
 989                                  p->expirationTime,
 990                                  p->anonymity,
 991                                  p->priority,
 992                                  pc->options,
 993                                  &publish_kblocks_cont,
 994                                  pc);
 995         }
 996       else
 997         {
 998           publish_kblocks_cont (pc,
 999                                 p->chk_uri,
1000                                 NULL);
1001         }
1002       return;
1003     }
1004   if ( (!p->is_directory) &&
1005        (p->data.file.do_index) )
1006     {
1007       if (NULL == p->filename)
1008         {
1009           p->data.file.do_index = GNUNET_NO;
1010           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1011                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1012                       "<no-name>",
1013                       _("needs to be an actual file"));
1014           GNUNET_FS_file_information_sync_ (p);
1015           publish_content (pc);
1016           return;
1017         }
1018       if (p->data.file.have_hash)
1019         {
1020           hash_for_index_cb (pc,
1021                              &p->data.file.file_id);
1022         }
1023       else
1024         {
1025           p->start_time = GNUNET_TIME_absolute_get ();
1026           GNUNET_CRYPTO_hash_file (pc->h->sched,
1027                                    GNUNET_SCHEDULER_PRIORITY_IDLE,
1028                                    p->filename,
1029                                    HASHING_BLOCKSIZE,
1030                                    &hash_for_index_cb,
1031                                    pc);
1032         }
1033       return;
1034     }
1035   publish_content (pc);
1036 }
1037
1038
1039 /**
1040  * Signal the FS's progress function that we are starting
1041  * an upload.
1042  *
1043  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1044  * @param fi the entry in the publish-structure
1045  * @param length length of the file or directory
1046  * @param meta metadata for the file or directory (can be modified)
1047  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1048  * @param anonymity pointer to selected anonymity level (can be modified)
1049  * @param priority pointer to selected priority (can be modified)
1050  * @param do_index should we index?
1051  * @param expirationTime pointer to selected expiration time (can be modified)
1052  * @param client_info pointer to client context set upon creation (can be modified)
1053  * @return GNUNET_OK to continue (always)
1054  */
1055 static int
1056 fip_signal_start(void *cls,
1057                  struct GNUNET_FS_FileInformation *fi,
1058                  uint64_t length,
1059                  struct GNUNET_CONTAINER_MetaData *meta,
1060                  struct GNUNET_FS_Uri **uri,
1061                  uint32_t *anonymity,
1062                  uint32_t *priority,
1063                  int *do_index,
1064                  struct GNUNET_TIME_Absolute *expirationTime,
1065                  void **client_info)
1066 {
1067   struct GNUNET_FS_PublishContext *sc = cls;
1068   struct GNUNET_FS_ProgressInfo pi;
1069   unsigned int kc;
1070   uint64_t left;
1071
1072   if (*do_index)
1073     {
1074       /* space for on-demand blocks */
1075       sc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1076     }
1077   else
1078     {
1079       /* space for DBlocks */
1080       sc->reserve_space += length;
1081     }
1082   /* entries for IBlocks and DBlocks, space for IBlocks */
1083   left = length;
1084   while (1)
1085     {
1086       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1087       sc->reserve_entries += left;
1088       if (left <= 1)
1089         break;
1090       left = left * sizeof (struct ContentHashKey);
1091       sc->reserve_space += left;
1092     }
1093   /* entries and space for keywords */
1094   if (NULL != *uri)
1095     {
1096       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1097       sc->reserve_entries += kc;
1098       sc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1099     }
1100   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1101   *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
1102   GNUNET_FS_file_information_sync_ (fi);
1103   return GNUNET_OK;
1104 }
1105
1106
1107 /**
1108  * Signal the FS's progress function that we are suspending
1109  * an upload.
1110  *
1111  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1112  * @param fi the entry in the publish-structure
1113  * @param length length of the file or directory
1114  * @param meta metadata for the file or directory (can be modified)
1115  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1116  * @param anonymity pointer to selected anonymity level (can be modified)
1117  * @param priority pointer to selected priority (can be modified)
1118  * @param do_index should we index?
1119  * @param expirationTime pointer to selected expiration time (can be modified)
1120  * @param client_info pointer to client context set upon creation (can be modified)
1121  * @return GNUNET_OK to continue (always)
1122  */
1123 static int
1124 fip_signal_suspend(void *cls,
1125                    struct GNUNET_FS_FileInformation *fi,
1126                    uint64_t length,
1127                    struct GNUNET_CONTAINER_MetaData *meta,
1128                    struct GNUNET_FS_Uri **uri,
1129                    uint32_t *anonymity,
1130                    uint32_t *priority,
1131                    int *do_index,
1132                    struct GNUNET_TIME_Absolute *expirationTime,
1133                    void **client_info)
1134 {
1135   struct GNUNET_FS_PublishContext*sc = cls;
1136   struct GNUNET_FS_ProgressInfo pi;
1137   uint64_t off;
1138
1139   GNUNET_free_non_null (fi->serialization);
1140   fi->serialization = NULL;
1141   off = (fi->chk_uri == NULL) ? 0 : length;
1142   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1143   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1144   *client_info = NULL;
1145   if (NULL != sc->dsh)
1146     {
1147       GNUNET_DATASTORE_disconnect (sc->dsh, GNUNET_NO);
1148       sc->dsh = NULL;
1149     }
1150   return GNUNET_OK;
1151 }
1152
1153
1154 /**
1155  * Create SUSPEND event for the given publish operation
1156  * and then clean up our state (without stop signal).
1157  *
1158  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1159  */
1160 void
1161 GNUNET_FS_publish_signal_suspend_ (void *cls)
1162 {
1163   struct GNUNET_FS_PublishContext *pc = cls;
1164
1165   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1166     {
1167       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1168       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1169     }
1170   GNUNET_FS_file_information_inspect (pc->fi,
1171                                       &fip_signal_suspend,
1172                                       pc);
1173   GNUNET_FS_end_top (pc->h, pc->top);
1174   publish_cleanup (pc, NULL);
1175 }
1176
1177
1178 /**
1179  * We have gotten a reply for our space reservation request.
1180  * Either fail (insufficient space) or start publishing for good.
1181  *
1182  * @param cls the 'struct GNUNET_FS_PublishContext*'
1183  * @param success positive reservation ID on success
1184  * @param msg error message on error, otherwise NULL
1185  */
1186 static void
1187 finish_reserve (void *cls,
1188                 int success,
1189                 const char *msg)
1190 {
1191   struct GNUNET_FS_PublishContext *pc = cls;
1192
1193   pc->qre = NULL;
1194   if ( (msg != NULL) ||
1195        (success <= 0) )
1196     {
1197       GNUNET_asprintf (&pc->fi->emsg,
1198                        _("Insufficient space for publishing: %s"),
1199                        msg);
1200       signal_publish_error (pc->fi,
1201                             pc,
1202                             pc->fi->emsg);
1203       return;
1204     }
1205   pc->rid = success;
1206   pc->upload_task
1207     = GNUNET_SCHEDULER_add_with_priority (pc->h->sched,
1208                                           GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1209                                           &GNUNET_FS_publish_main_,
1210                                           pc);
1211 }
1212
1213
1214 /**
1215  * Publish a file or directory.
1216  *
1217  * @param h handle to the file sharing subsystem
1218  * @param fi information about the file or directory structure to publish
1219  * @param namespace namespace to publish the file in, NULL for no namespace
1220  * @param nid identifier to use for the publishd content in the namespace
1221  *        (can be NULL, must be NULL if namespace is NULL)
1222  * @param nuid update-identifier that will be used for future updates
1223  *        (can be NULL, must be NULL if namespace or nid is NULL)
1224  * @param options options for the publication
1225  * @return context that can be used to control the publish operation
1226  */
1227 struct GNUNET_FS_PublishContext *
1228 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1229                          struct GNUNET_FS_FileInformation *fi,
1230                          struct GNUNET_FS_Namespace *namespace,
1231                          const char *nid,
1232                          const char *nuid,
1233                          enum GNUNET_FS_PublishOptions options)
1234 {
1235   struct GNUNET_FS_PublishContext *ret;
1236   struct GNUNET_DATASTORE_Handle *dsh;
1237
1238   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1239     {
1240       dsh = GNUNET_DATASTORE_connect (h->cfg,
1241                                       h->sched);
1242       if (NULL == dsh)
1243         return NULL;
1244     }
1245   else
1246     {
1247       dsh = NULL;
1248     }
1249   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1250   ret->dsh = dsh;
1251   ret->h = h;
1252   ret->fi = fi;
1253   ret->namespace = namespace;
1254   if (namespace != NULL)
1255     {
1256       namespace->rc++;
1257       GNUNET_assert (NULL != nid);
1258       ret->nid = GNUNET_strdup (nid);
1259       if (NULL != nuid)
1260         ret->nuid = GNUNET_strdup (nuid);
1261     }
1262   /* signal start */
1263   GNUNET_FS_file_information_inspect (ret->fi,
1264                                       &fip_signal_start,
1265                                       ret);
1266   ret->fi_pos = ret->fi;
1267   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1268   GNUNET_FS_publish_sync_ (ret);
1269   if (NULL != ret->dsh)
1270     {
1271       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1272                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1273                   (unsigned int) ret->reserve_entries,
1274                   (unsigned long long) ret->reserve_space);
1275       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1276                                            ret->reserve_space,
1277                                            ret->reserve_entries,
1278                                            UINT_MAX,
1279                                            UINT_MAX,
1280                                            GNUNET_TIME_UNIT_FOREVER_REL,
1281                                            &finish_reserve,
1282                                            ret);
1283     }
1284   else
1285     {
1286       ret->upload_task
1287         = GNUNET_SCHEDULER_add_with_priority (h->sched,
1288                                               GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1289                                               &GNUNET_FS_publish_main_,
1290                                               ret);
1291     }
1292   return ret;
1293 }
1294
1295
1296 /**
1297  * Signal the FS's progress function that we are stopping
1298  * an upload.
1299  *
1300  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1301  * @param fi the entry in the publish-structure
1302  * @param length length of the file or directory
1303  * @param meta metadata for the file or directory (can be modified)
1304  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1305  * @param anonymity pointer to selected anonymity level (can be modified)
1306  * @param priority pointer to selected priority (can be modified)
1307  * @param do_index should we index?
1308  * @param expirationTime pointer to selected expiration time (can be modified)
1309  * @param client_info pointer to client context set upon creation (can be modified)
1310  * @return GNUNET_OK to continue (always)
1311  */
1312 static int
1313 fip_signal_stop(void *cls,
1314                 struct GNUNET_FS_FileInformation *fi,
1315                 uint64_t length,
1316                 struct GNUNET_CONTAINER_MetaData *meta,
1317                 struct GNUNET_FS_Uri **uri,
1318                 uint32_t *anonymity,
1319                 uint32_t *priority,
1320                 int *do_index,
1321                 struct GNUNET_TIME_Absolute *expirationTime,
1322                 void **client_info)
1323 {
1324   struct GNUNET_FS_PublishContext*sc = cls;
1325   struct GNUNET_FS_ProgressInfo pi;
1326   uint64_t off;
1327
1328   if (fi->serialization != NULL)
1329     {
1330       GNUNET_FS_remove_sync_file_ (sc->h,
1331                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1332                                    fi->serialization);
1333       GNUNET_free (fi->serialization);
1334       fi->serialization = NULL;
1335     }
1336   off = (fi->chk_uri == NULL) ? 0 : length;
1337   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1338   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, sc, fi, off));
1339   *client_info = NULL;
1340   return GNUNET_OK;
1341 }
1342
1343
1344 /**
1345  * Stop an upload.  Will abort incomplete uploads (but
1346  * not remove blocks that have already been publishd) or
1347  * simply clean up the state for completed uploads.
1348  * Must NOT be called from within the event callback!
1349  *
1350  * @param pc context for the upload to stop
1351  */
1352 void
1353 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1354 {
1355   GNUNET_FS_end_top (pc->h, pc->top);
1356   if (NULL != pc->qre)
1357     {
1358       GNUNET_DATASTORE_cancel (pc->qre);
1359       pc->qre = NULL;
1360     }
1361   if (NULL != pc->dsh)
1362     {
1363       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1364       pc->dsh = NULL;
1365     }
1366   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1367     {
1368       GNUNET_SCHEDULER_cancel (pc->h->sched, pc->upload_task);
1369       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1370     }
1371   if (pc->serialization != NULL)
1372     {
1373       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1374       GNUNET_free (pc->serialization);
1375       pc->serialization = NULL;
1376     }
1377   GNUNET_FS_file_information_inspect (pc->fi,
1378                                       &fip_signal_stop,
1379                                       pc);
1380   if (GNUNET_YES == pc->in_network_wait)
1381     {
1382       pc->in_network_wait = GNUNET_SYSERR;
1383       return;
1384     }
1385   publish_cleanup (pc, NULL);
1386 }
1387
1388
1389 /**
1390  * Context for the KSK publication.
1391  */
1392 struct PublishKskContext
1393 {
1394
1395   /**
1396    * Keywords to use.
1397    */
1398   struct GNUNET_FS_Uri *ksk_uri;
1399
1400   /**
1401    * Global FS context.
1402    */
1403   struct GNUNET_FS_Handle *h;
1404
1405   /**
1406    * The master block that we are sending
1407    * (in plaintext), has "mdsize+slen" more
1408    * bytes than the struct would suggest.
1409    */
1410   struct KBlock *kb;
1411
1412   /**
1413    * Buffer of the same size as "kb" for
1414    * the encrypted version.
1415    */
1416   struct KBlock *cpy;
1417
1418   /**
1419    * Handle to the datastore, NULL if we are just
1420    * simulating.
1421    */
1422   struct GNUNET_DATASTORE_Handle *dsh;
1423
1424   /**
1425    * Function to call once we're done.
1426    */
1427   GNUNET_FS_PublishContinuation cont;
1428
1429   /**
1430    * Closure for cont.
1431    */
1432   void *cont_cls;
1433
1434   /**
1435    * When should the KBlocks expire?
1436    */
1437   struct GNUNET_TIME_Absolute expirationTime;
1438
1439   /**
1440    * Size of the serialized metadata.
1441    */
1442   ssize_t mdsize;
1443
1444   /**
1445    * Size of the (CHK) URI as a string.
1446    */
1447   size_t slen;
1448
1449   /**
1450    * Keyword that we are currently processing.
1451    */
1452   unsigned int i;
1453
1454   /**
1455    * Anonymity level for the KBlocks.
1456    */
1457   uint32_t anonymity;
1458
1459   /**
1460    * Priority for the KBlocks.
1461    */
1462   uint32_t priority;
1463 };
1464
1465
1466 /**
1467  * Continuation of "GNUNET_FS_publish_ksk" that performs
1468  * the actual publishing operation (iterating over all
1469  * of the keywords).
1470  *
1471  * @param cls closure of type "struct PublishKskContext*"
1472  * @param tc unused
1473  */
1474 static void
1475 publish_ksk_cont (void *cls,
1476                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1477
1478
1479 /**
1480  * Function called by the datastore API with
1481  * the result from the PUT request.
1482  *
1483  * @param cls closure of type "struct PublishKskContext*"
1484  * @param success GNUNET_OK on success
1485  * @param msg error message (or NULL)
1486  */
1487 static void
1488 kb_put_cont (void *cls,
1489              int success,
1490              const char *msg)
1491 {
1492   struct PublishKskContext *pkc = cls;
1493
1494   if (GNUNET_OK != success)
1495     {
1496       if (NULL != pkc->dsh)
1497         {
1498           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1499           pkc->dsh = NULL;
1500         }
1501       GNUNET_free (pkc->cpy);
1502       GNUNET_free (pkc->kb);
1503       pkc->cont (pkc->cont_cls,
1504                  NULL,
1505                  msg);
1506       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1507       GNUNET_free (pkc);
1508       return;
1509     }
1510   GNUNET_SCHEDULER_add_continuation (pkc->h->sched,
1511                                      &publish_ksk_cont,
1512                                      pkc,
1513                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1514 }
1515
1516
1517 /**
1518  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1519  * publishing operation (iterating over all of the keywords).
1520  *
1521  * @param cls closure of type "struct PublishKskContext*"
1522  * @param tc unused
1523  */
1524 static void
1525 publish_ksk_cont (void *cls,
1526                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1527 {
1528   struct PublishKskContext *pkc = cls;
1529   const char *keyword;
1530   GNUNET_HashCode key;
1531   GNUNET_HashCode query;
1532   struct GNUNET_CRYPTO_AesSessionKey skey;
1533   struct GNUNET_CRYPTO_AesInitializationVector iv;
1534   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1535
1536
1537   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1538        (NULL == pkc->dsh) )
1539     {
1540       if (NULL != pkc->dsh)
1541         {
1542           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1543           pkc->dsh = NULL;
1544         }
1545       GNUNET_free (pkc->cpy);
1546       GNUNET_free (pkc->kb);
1547       pkc->cont (pkc->cont_cls,
1548                  pkc->ksk_uri,
1549                  NULL);
1550       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1551       GNUNET_free (pkc);
1552       return;
1553     }
1554   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1555   /* first character of keyword indicates if it is
1556      mandatory or not -- ignore for hashing */
1557   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1558   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1559   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1560                              pkc->slen + pkc->mdsize,
1561                              &skey,
1562                              &iv,
1563                              &pkc->cpy[1]);
1564   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1565   GNUNET_assert (NULL != pk);
1566   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1567   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1568                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1569                       &query);
1570   GNUNET_assert (GNUNET_OK ==
1571                  GNUNET_CRYPTO_rsa_sign (pk,
1572                                          &pkc->cpy->purpose,
1573                                          &pkc->cpy->signature));
1574   GNUNET_CRYPTO_rsa_key_free (pk);
1575   GNUNET_DATASTORE_put (pkc->dsh,
1576                         0,
1577                         &query,
1578                         pkc->mdsize +
1579                         sizeof (struct KBlock) +
1580                         pkc->slen,
1581                         pkc->cpy,
1582                         GNUNET_BLOCK_TYPE_KBLOCK,
1583                         pkc->priority,
1584                         pkc->anonymity,
1585                         pkc->expirationTime,
1586                         -2, 1,
1587                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1588                         &kb_put_cont,
1589                         pkc);
1590 }
1591
1592
1593 /**
1594  * Publish a CHK under various keywords on GNUnet.
1595  *
1596  * @param h handle to the file sharing subsystem
1597  * @param ksk_uri keywords to use
1598  * @param meta metadata to use
1599  * @param uri URI to refer to in the KBlock
1600  * @param expirationTime when the KBlock expires
1601  * @param anonymity anonymity level for the KBlock
1602  * @param priority priority for the KBlock
1603  * @param options publication options
1604  * @param cont continuation
1605  * @param cont_cls closure for cont
1606  */
1607 void
1608 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1609                        const struct GNUNET_FS_Uri *ksk_uri,
1610                        const struct GNUNET_CONTAINER_MetaData *meta,
1611                        const struct GNUNET_FS_Uri *uri,
1612                        struct GNUNET_TIME_Absolute expirationTime,
1613                        uint32_t anonymity,
1614                        uint32_t priority,
1615                        enum GNUNET_FS_PublishOptions options,
1616                        GNUNET_FS_PublishContinuation cont,
1617                        void *cont_cls)
1618 {
1619   struct PublishKskContext *pkc;
1620   char *uris;
1621   size_t size;
1622   char *kbe;
1623   char *sptr;
1624
1625   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1626   pkc->h = h;
1627   pkc->expirationTime = expirationTime;
1628   pkc->anonymity = anonymity;
1629   pkc->priority = priority;
1630   pkc->cont = cont;
1631   pkc->cont_cls = cont_cls;
1632   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1633     {
1634       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg,
1635                                            h->sched);
1636       if (pkc->dsh == NULL)
1637         {
1638           cont (cont_cls, NULL, _("Could not connect to datastore."));
1639           GNUNET_free (pkc);
1640           return;
1641         }
1642     }
1643   if (meta == NULL)
1644     pkc->mdsize = 0;
1645   else
1646     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1647   GNUNET_assert (pkc->mdsize >= 0);
1648   uris = GNUNET_FS_uri_to_string (uri);
1649   pkc->slen = strlen (uris) + 1;
1650   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1651   if (size > MAX_KBLOCK_SIZE)
1652     {
1653       size = MAX_KBLOCK_SIZE;
1654       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1655     }
1656   pkc->kb = GNUNET_malloc (size);
1657   kbe = (char *) &pkc->kb[1];
1658   memcpy (kbe, uris, pkc->slen);
1659   GNUNET_free (uris);
1660   sptr = &kbe[pkc->slen];
1661   if (meta != NULL)
1662     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1663                                                         &sptr,
1664                                                         pkc->mdsize,
1665                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1666   if (pkc->mdsize == -1)
1667     {
1668       GNUNET_break (0);
1669       GNUNET_free (pkc->kb);
1670       if (pkc->dsh != NULL)
1671         {
1672           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1673           pkc->dsh = NULL;
1674         }
1675       cont (cont_cls, NULL, _("Internal error."));
1676       GNUNET_free (pkc);
1677       return;
1678     }
1679   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1680
1681   pkc->cpy = GNUNET_malloc (size);
1682   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1683                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1684                                   pkc->mdsize +
1685                                   pkc->slen);
1686   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1687   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1688   GNUNET_SCHEDULER_add_continuation (h->sched,
1689                                      &publish_ksk_cont,
1690                                      pkc,
1691                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1692 }
1693
1694
1695 /**
1696  * Context for the SKS publication.
1697  */
1698 struct PublishSksContext
1699 {
1700
1701   /**
1702    * Global FS context.
1703    */
1704   struct GNUNET_FS_Uri *uri;
1705
1706   /**
1707    * Handle to the datastore.
1708    */
1709   struct GNUNET_DATASTORE_Handle *dsh;
1710
1711   /**
1712    * Function to call once we're done.
1713    */
1714   GNUNET_FS_PublishContinuation cont;
1715
1716   /**
1717    * Closure for cont.
1718    */
1719   void *cont_cls;
1720
1721 };
1722
1723
1724 /**
1725  * Function called by the datastore API with
1726  * the result from the PUT (SBlock) request.
1727  *
1728  * @param cls closure of type "struct PublishSksContext*"
1729  * @param success GNUNET_OK on success
1730  * @param msg error message (or NULL)
1731  */
1732 static void
1733 sb_put_cont (void *cls,
1734              int success,
1735              const char *msg)
1736 {
1737   struct PublishSksContext *psc = cls;
1738
1739   if (NULL != psc->dsh)
1740     {
1741       GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO);
1742       psc->dsh = NULL;
1743     }
1744   if (GNUNET_OK != success)
1745     psc->cont (psc->cont_cls,
1746                NULL,
1747                msg);
1748   else
1749     psc->cont (psc->cont_cls,
1750                psc->uri,
1751                NULL);
1752   GNUNET_FS_uri_destroy (psc->uri);
1753   GNUNET_free (psc);
1754 }
1755
1756
1757 /**
1758  * Publish an SBlock on GNUnet.
1759  *
1760  * @param h handle to the file sharing subsystem
1761  * @param namespace namespace to publish in
1762  * @param identifier identifier to use
1763  * @param update update identifier to use
1764  * @param meta metadata to use
1765  * @param uri URI to refer to in the SBlock
1766  * @param expirationTime when the SBlock expires
1767  * @param anonymity anonymity level for the SBlock
1768  * @param priority priority for the SBlock
1769  * @param options publication options
1770  * @param cont continuation
1771  * @param cont_cls closure for cont
1772  */
1773 void
1774 GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
1775                        struct GNUNET_FS_Namespace *namespace,
1776                        const char *identifier,
1777                        const char *update,
1778                        const struct GNUNET_CONTAINER_MetaData *meta,
1779                        const struct GNUNET_FS_Uri *uri,
1780                        struct GNUNET_TIME_Absolute expirationTime,
1781                        uint32_t anonymity,
1782                        uint32_t priority,
1783                        enum GNUNET_FS_PublishOptions options,
1784                        GNUNET_FS_PublishContinuation cont,
1785                        void *cont_cls)
1786 {
1787   struct PublishSksContext *psc;
1788   struct GNUNET_CRYPTO_AesSessionKey sk;
1789   struct GNUNET_CRYPTO_AesInitializationVector iv;
1790   struct GNUNET_FS_Uri *sks_uri;
1791   char *uris;
1792   size_t size;
1793   size_t slen;
1794   size_t nidlen;
1795   size_t idlen;
1796   ssize_t mdsize;
1797   struct SBlock *sb;
1798   struct SBlock *sb_enc;
1799   char *dest;
1800   struct GNUNET_CONTAINER_MetaData *mmeta;
1801   GNUNET_HashCode key;         /* hash of thisId = key */
1802   GNUNET_HashCode id;          /* hash of hc = identifier */
1803   GNUNET_HashCode query;       /* id ^ nsid = DB query */
1804
1805   if (NULL == meta)
1806     mmeta = GNUNET_CONTAINER_meta_data_create ();
1807   else
1808     mmeta = GNUNET_CONTAINER_meta_data_duplicate (meta);
1809   uris = GNUNET_FS_uri_to_string (uri);
1810   slen = strlen (uris) + 1;
1811   idlen = strlen (identifier);
1812   if (update == NULL)
1813     update = "";
1814   nidlen = strlen (update) + 1;
1815   mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (mmeta);
1816   size = sizeof (struct SBlock) + slen + nidlen + mdsize;
1817   if (size > MAX_SBLOCK_SIZE)
1818     {
1819       size = MAX_SBLOCK_SIZE;
1820       mdsize = size - (sizeof (struct SBlock) + slen + nidlen);
1821     }
1822   sb = GNUNET_malloc (sizeof (struct SBlock) + size);
1823   dest = (char *) &sb[1];
1824   memcpy (dest, update, nidlen);
1825   dest += nidlen;
1826   memcpy (dest, uris, slen);
1827   GNUNET_free (uris);
1828   dest += slen;
1829   mdsize = GNUNET_CONTAINER_meta_data_serialize (mmeta,
1830                                                  &dest,
1831                                                  mdsize,
1832                                                  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1833   GNUNET_CONTAINER_meta_data_destroy (mmeta);
1834   if (mdsize == -1)
1835     {
1836       GNUNET_break (0);
1837       GNUNET_free (sb);
1838       cont (cont_cls,
1839             NULL,
1840             _("Internal error."));
1841       return;
1842     }
1843   size = sizeof (struct SBlock) + mdsize + slen + nidlen;
1844   sb_enc = GNUNET_malloc (size);
1845   GNUNET_CRYPTO_hash (identifier, idlen, &key);
1846   GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &id);
1847   sks_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri));
1848   sks_uri->type = sks;
1849   GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &sb_enc->subspace);
1850   GNUNET_CRYPTO_hash (&sb_enc->subspace,
1851                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1852                       &sks_uri->data.sks.namespace);
1853   sks_uri->data.sks.identifier = GNUNET_strdup (identifier);
1854   GNUNET_CRYPTO_hash_xor (&id,
1855                           &sks_uri->data.sks.namespace,
1856                           &sb_enc->identifier);
1857   GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv);
1858   GNUNET_CRYPTO_aes_encrypt (&sb[1],
1859                              size - sizeof (struct SBlock),
1860                              &sk,
1861                              &iv,
1862                              &sb_enc[1]);
1863   sb_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK);
1864   sb_enc->purpose.size = htonl(slen + mdsize + nidlen
1865                                + sizeof(struct SBlock)
1866                                - sizeof(struct GNUNET_CRYPTO_RsaSignature));
1867   GNUNET_assert (GNUNET_OK ==
1868                  GNUNET_CRYPTO_rsa_sign (namespace->key,
1869                                          &sb_enc->purpose,
1870                                          &sb_enc->signature));
1871   psc = GNUNET_malloc (sizeof(struct PublishSksContext));
1872   psc->uri = sks_uri;
1873   psc->cont = cont;
1874   psc->cont_cls = cont_cls;
1875   if (0 != (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1876     {
1877       GNUNET_free (sb_enc);
1878       GNUNET_free (sb);
1879       sb_put_cont (psc,
1880                    GNUNET_OK,
1881                    NULL);
1882       return;
1883     }
1884   psc->dsh = GNUNET_DATASTORE_connect (h->cfg, h->sched);
1885   if (NULL == psc->dsh)
1886     {
1887       GNUNET_free (sb_enc);
1888       GNUNET_free (sb);
1889       sb_put_cont (psc,
1890                    GNUNET_NO,
1891                    _("Failed to connect to datastore."));
1892       return;
1893     }
1894   GNUNET_CRYPTO_hash_xor (&sks_uri->data.sks.namespace,
1895                           &id,
1896                           &query);
1897   GNUNET_DATASTORE_put (psc->dsh,
1898                         0,
1899                         &sb_enc->identifier,
1900                         size,
1901                         sb_enc,
1902                         GNUNET_BLOCK_TYPE_SBLOCK,
1903                         priority,
1904                         anonymity,
1905                         expirationTime,
1906                         -2, 1,
1907                         GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1908                         &sb_put_cont,
1909                         psc);
1910   GNUNET_free (sb);
1911   GNUNET_free (sb_enc);
1912 }
1913
1914 /* end of fs_publish.c */