src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see https://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Fill in all of the generic fields for
  42  * a publish event and call the callback.
  43  *
  44  * @param pi structure to fill in
  45  * @param pc overall publishing context
  46  * @param p file information for the file being published
  47  * @param offset where in the file are we so far
  48  * @return value returned from callback
  49  */
  50 void *
  51 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  52                                 struct GNUNET_FS_PublishContext *pc,
  53                                 const struct GNUNET_FS_FileInformation *p,
  54                                 uint64_t offset)
  55 {
  56   pi->value.publish.pc = pc;
  57   pi->value.publish.fi = p;
  58   pi->value.publish.cctx
  59     = p->client_info;
  60   pi->value.publish.pctx
  61     = (NULL == p->dir) ? NULL : p->dir->client_info;
  62   pi->value.publish.filename = p->filename;
  63   pi->value.publish.size
  64     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  65   pi->value.publish.eta
  66     = GNUNET_TIME_calculate_eta (p->start_time,
  67                                  offset,
  68                                  pi->value.publish.size);
  69   pi->value.publish.completed = offset;
  70   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  71   pi->value.publish.anonymity = p->bo.anonymity_level;
  72   return pc->h->upcb (pc->h->upcb_cls,
  73                       pi);
  74 }
  75
  76
  77 /**
  78  * Cleanup the publish context, we're done with it.
  79  *
  80  * @param cls struct to clean up after
  81  * @param tc scheduler context
  82  */
  83 static void
  84 publish_cleanup (void *cls,
  85                  const struct GNUNET_SCHEDULER_TaskContext *tc)
  86 {
  87   struct GNUNET_FS_PublishContext *pc = cls;
  88
  89 #if DEBUG_PUBLISH
  90   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
  91               "Cleaning up publish context (done!)\n");
  92 #endif
  93   if (pc->fhc != NULL)
  94     {
  95       GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
  96       pc->fhc = NULL;
  97     }
  98   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
  99   if (pc->namespace != NULL)
 100     {
 101       GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 102       pc->namespace = NULL;
 103     }
 104   GNUNET_free_non_null (pc->nid);
 105   GNUNET_free_non_null (pc->nuid);
 106   GNUNET_free_non_null (pc->serialization);
 107   if (pc->dsh != NULL)
 108     {
 109       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 110       pc->dsh = NULL;
 111     }
 112   if (pc->client != NULL)
 113     {
 114       GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 115       pc->client = NULL;
 116     }
 117   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 118   GNUNET_free (pc);
 119 }
 120
 121
 122 /**
 123  * Function called by the datastore API with
 124  * the result from the PUT request.
 125  *
 126  * @param cls the 'struct GNUNET_FS_PublishContext'
 127  * @param success GNUNET_OK on success
 128  * @param msg error message (or NULL)
 129  */
 130 static void
 131 ds_put_cont (void *cls,
 132              int success,
 133              const char *msg)
 134 {
 135   struct GNUNET_FS_PublishContext *pc = cls;
 136   struct GNUNET_FS_ProgressInfo pi;
 137
 138   pc->qre = NULL;
 139   if (GNUNET_SYSERR == pc->in_network_wait)
 140     {
 141       /* we were aborted in the meantime, finish shutdown! */
 142       GNUNET_SCHEDULER_add_continuation (&publish_cleanup,
 143                                          pc,
 144                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 145       return;
 146     }
 147   GNUNET_assert (GNUNET_YES == pc->in_network_wait);
 148   pc->in_network_wait = GNUNET_NO;
 149   if (GNUNET_SYSERR == success)
 150     {
 151       GNUNET_asprintf (&pc->fi_pos->emsg,
 152                        _("Publishing failed: %s"),
 153                        msg);
 154       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 155       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 156       pi.value.publish.specifics.error.message = pc->fi_pos->emsg;
 157       pc->fi_pos->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0);
 158       if ( (pc->fi_pos->is_directory == GNUNET_NO) &&
 159            (pc->fi_pos->filename != NULL) &&
 160            (pc->fi_pos->data.file.do_index == GNUNET_YES) )
 161         {
 162           /* run unindex to clean up */
 163           GNUNET_FS_unindex_start (pc->h,
 164                                    pc->fi_pos->filename,
 165                                    NULL);
 166         }
 167     }
 168   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 169   pc->upload_task
 170     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 171                                           &GNUNET_FS_publish_main_,
 172                                           pc);
 173 }
 174
 175
 176 /**
 177  * Generate the callback that signals clients
 178  * that a file (or directory) has been completely
 179  * published.
 180  *
 181  * @param p the completed upload
 182  * @param pc context of the publication
 183  */
 184 static void
 185 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 186                            struct GNUNET_FS_PublishContext *pc)
 187 {
 188   struct GNUNET_FS_ProgressInfo pi;
 189
 190   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 191   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 192   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 193   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p,
 194                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 195 }
 196
 197
 198 /**
 199  * Generate the callback that signals clients
 200  * that a file (or directory) has encountered
 201  * a problem during publication.
 202  *
 203  * @param p the upload that had trouble
 204  * @param pc context of the publication
 205  * @param emsg error message
 206  */
 207 static void
 208 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 209                       struct GNUNET_FS_PublishContext *pc,
 210                       const char *emsg)
 211 {
 212   struct GNUNET_FS_ProgressInfo pi;
 213
 214   p->emsg = GNUNET_strdup (emsg);
 215   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 216   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 217   pi.value.publish.specifics.error.message =emsg;
 218   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 219   if ( (p->is_directory == GNUNET_NO) &&
 220        (p->filename != NULL) &&
 221        (p->data.file.do_index == GNUNET_YES) )
 222     {
 223       /* run unindex to clean up */
 224       GNUNET_FS_unindex_start (pc->h,
 225                                p->filename,
 226                                NULL);
 227     }
 228
 229 }
 230
 231
 232 /**
 233  * Datastore returns from reservation cancel request.
 234  *
 235  * @param cls the 'struct GNUNET_FS_PublishContext'
 236  * @param success success code (not used)
 237  * @param msg error message (typically NULL, not used)
 238  */
 239 static void
 240 finish_release_reserve (void *cls,
 241                         int success,
 242                         const char *msg)
 243 {
 244   struct GNUNET_FS_PublishContext *pc = cls;
 245
 246   pc->qre = NULL;
 247 #if DEBUG_PUBLISH
 248   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 249               "Releasing reserve done!\n");
 250 #endif
 251   signal_publish_completion (pc->fi, pc);
 252   pc->all_done = GNUNET_YES;
 253   GNUNET_FS_publish_sync_ (pc);
 254 }
 255
 256
 257 /**
 258  * We've finished publishing the SBlock as part of a larger upload.
 259  * Check the result and complete the larger upload.
 260  *
 261  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 262  * @param uri URI of the published SBlock
 263  * @param emsg NULL on success, otherwise error message
 264  */
 265 static void
 266 publish_sblocks_cont (void *cls,
 267                       const struct GNUNET_FS_Uri *uri,
 268                       const char *emsg)
 269 {
 270   struct GNUNET_FS_PublishContext *pc = cls;
 271   if (NULL != emsg)
 272     {
 273       signal_publish_error (pc->fi,
 274                             pc,
 275                             emsg);
 276       GNUNET_FS_publish_sync_ (pc);
 277       return;
 278     }
 279   GNUNET_assert (pc->qre == NULL);
 280   if ( (pc->dsh != NULL) &&
 281        (pc->rid != 0) )
 282     {
 283       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 284                                                   pc->rid,
 285                                                   UINT_MAX,
 286                                                   UINT_MAX,
 287                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 288                                                   &finish_release_reserve,
 289                                                   pc);
 290     }
 291   else
 292     {
 293       finish_release_reserve (pc, GNUNET_OK, NULL);
 294     }
 295 }
 296
 297
 298 /**
 299  * We are almost done publishing the structure,
 300  * add SBlocks (if needed).
 301  *
 302  * @param pc overall upload data
 303  */
 304 static void
 305 publish_sblock (struct GNUNET_FS_PublishContext *pc)
 306 {
 307   if (NULL != pc->namespace)
 308     GNUNET_FS_publish_sks (pc->h,
 309                            pc->namespace,
 310                            pc->nid,
 311                            pc->nuid,
 312                            pc->fi->meta,
 313                            pc->fi->chk_uri,
 314                            &pc->fi->bo,
 315                            pc->options,
 316                            &publish_sblocks_cont,
 317                            pc);
 318   else
 319     publish_sblocks_cont (pc, NULL, NULL);
 320 }
 321
 322
 323 /**
 324  * We've finished publishing a KBlock as part of a larger upload.
 325  * Check the result and continue the larger upload.
 326  *
 327  * @param cls the "struct GNUNET_FS_PublishContext*"
 328  *        of the larger upload
 329  * @param uri URI of the published blocks
 330  * @param emsg NULL on success, otherwise error message
 331  */
 332 static void
 333 publish_kblocks_cont (void *cls,
 334                       const struct GNUNET_FS_Uri *uri,
 335                       const char *emsg)
 336 {
 337   struct GNUNET_FS_PublishContext *pc = cls;
 338   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 339
 340   if (NULL != emsg)
 341     {
 342 #if DEBUG_PUBLISH
 343       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 344                   "Error uploading KSK blocks: %s\n",
 345                   emsg);
 346 #endif
 347       signal_publish_error (p, pc, emsg);
 348       GNUNET_FS_file_information_sync_ (p);
 349       GNUNET_FS_publish_sync_ (pc);
 350       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 351       pc->upload_task
 352         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 353                                               &GNUNET_FS_publish_main_,
 354                                               pc);
 355       return;
 356     }
 357 #if DEBUG_PUBLISH
 358   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 359               "KSK blocks published, moving on to next file\n");
 360 #endif
 361   if (NULL != p->dir)
 362     signal_publish_completion (p, pc);
 363   /* move on to next file */
 364   if (NULL != p->next)
 365     pc->fi_pos = p->next;
 366   else
 367     pc->fi_pos = p->dir;
 368   GNUNET_FS_publish_sync_ (pc);
 369   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 370   pc->upload_task
 371     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 372                                           &GNUNET_FS_publish_main_,
 373                                           pc);
 374 }
 375
 376
 377 /**
 378  * Function called by the tree encoder to obtain
 379  * a block of plaintext data (for the lowest level
 380  * of the tree).
 381  *
 382  * @param cls our publishing context
 383  * @param offset identifies which block to get
 384  * @param max (maximum) number of bytes to get; returning
 385  *        fewer will also cause errors
 386  * @param buf where to copy the plaintext buffer
 387  * @param emsg location to store an error message (on error)
 388  * @return number of bytes copied to buf, 0 on error
 389  */
 390 static size_t
 391 block_reader (void *cls,
 392               uint64_t offset,
 393               size_t max,
 394               void *buf,
 395               char **emsg)
 396 {
 397   struct GNUNET_FS_PublishContext *pc = cls;
 398   struct GNUNET_FS_FileInformation *p;
 399   size_t pt_size;
 400   const char *dd;
 401
 402   p = pc->fi_pos;
 403   if (p->is_directory)
 404     {
 405       pt_size = GNUNET_MIN(max,
 406                            p->data.dir.dir_size - offset);
 407       dd = p->data.dir.dir_data;
 408       memcpy (buf,
 409               &dd[offset],
 410               pt_size);
 411     }
 412   else
 413     {
 414       pt_size = GNUNET_MIN(max,
 415                            p->data.file.file_size - offset);
 416       if (pt_size == 0)
 417         return 0; /* calling reader with pt_size==0
 418                      might free buf, so don't! */
 419       if (pt_size !=
 420           p->data.file.reader (p->data.file.reader_cls,
 421                                offset,
 422                                pt_size,
 423                                buf,
 424                                emsg))
 425         return 0;
 426     }
 427   return pt_size;
 428 }
 429
 430
 431 /**
 432  * The tree encoder has finished processing a
 433  * file.   Call it's finish method and deal with
 434  * the final result.
 435  *
 436  * @param cls our publishing context
 437  * @param tc scheduler's task context (not used)
 438  */
 439 static void
 440 encode_cont (void *cls,
 441              const struct GNUNET_SCHEDULER_TaskContext *tc)
 442 {
 443   struct GNUNET_FS_PublishContext *pc = cls;
 444   struct GNUNET_FS_FileInformation *p;
 445   struct GNUNET_FS_ProgressInfo pi;
 446   char *emsg;
 447   uint64_t flen;
 448
 449   p = pc->fi_pos;
 450   GNUNET_FS_tree_encoder_finish (p->te,
 451                                  &p->chk_uri,
 452                                  &emsg);
 453   p->te = NULL;
 454   if (NULL != emsg)
 455     {
 456 #if DEBUG_PUBLISH
 457       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 458                   "Error during tree walk: %s\n",
 459                   emsg);
 460 #endif
 461       GNUNET_asprintf (&p->emsg,
 462                        _("Publishing failed: %s"),
 463                        emsg);
 464       GNUNET_free (emsg);
 465       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 466       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 467       pi.value.publish.specifics.error.message = p->emsg;
 468       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 469     }
 470 #if DEBUG_PUBLISH
 471   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 472               "Finished with tree encoder\n");
 473 #endif
 474   /* final progress event */
 475   flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
 476   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 477   pi.value.publish.specifics.progress.data = NULL;
 478   pi.value.publish.specifics.progress.offset = flen;
 479   pi.value.publish.specifics.progress.data_len = 0;
 480   pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
 481   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen);
 482
 483   /* continue with main */
 484   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 485   pc->upload_task
 486     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 487                                           &GNUNET_FS_publish_main_,
 488                                           pc);
 489 }
 490
 491
 492 /**
 493  * Function called asking for the current (encoded)
 494  * block to be processed.  After processing the
 495  * client should either call "GNUNET_FS_tree_encode_next"
 496  * or (on error) "GNUNET_FS_tree_encode_finish".
 497  *
 498  * @param cls closure
 499  * @param chk content hash key for the block
 500  * @param offset offset of the block in the file
 501  * @param depth depth of the block in the file, 0 for DBLOCK
 502  * @param type type of the block (IBLOCK or DBLOCK)
 503  * @param block the (encrypted) block
 504  * @param block_size size of block (in bytes)
 505  */
 506 static void
 507 block_proc (void *cls,
 508             const struct ContentHashKey *chk,
 509             uint64_t offset,
 510             unsigned int depth,
 511             enum GNUNET_BLOCK_Type type,
 512             const void *block,
 513             uint16_t block_size)
 514 {
 515   struct GNUNET_FS_PublishContext *pc = cls;
 516   struct GNUNET_FS_FileInformation *p;
 517   struct OnDemandBlock odb;
 518
 519   p = pc->fi_pos;
 520   if (NULL == pc->dsh)
 521     {
 522 #if DEBUG_PUBLISH
 523       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 524                   "Waiting for datastore connection\n");
 525 #endif
 526       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 527       pc->upload_task
 528         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 529                                               &GNUNET_FS_publish_main_,
 530                                               pc);
 531       return;
 532     }
 533
 534   GNUNET_assert (GNUNET_NO == pc->in_network_wait);
 535   pc->in_network_wait = GNUNET_YES;
 536   if ( (! p->is_directory) &&
 537        (GNUNET_YES == p->data.file.do_index) &&
 538        (type == GNUNET_BLOCK_TYPE_FS_DBLOCK) )
 539     {
 540 #if DEBUG_PUBLISH
 541       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 542                   "Indexing block `%s' for offset %llu with index size %u\n",
 543                   GNUNET_h2s (&chk->query),
 544                   (unsigned long long) offset,
 545                   sizeof (struct OnDemandBlock));
 546 #endif
 547       odb.offset = GNUNET_htonll (offset);
 548       odb.file_id = p->data.file.file_id;
 549       GNUNET_assert (pc->qre == NULL);
 550       pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 551                                       (p->is_directory) ? 0 : pc->rid,
 552                                       &chk->query,
 553                                       sizeof (struct OnDemandBlock),
 554                                       &odb,
 555                                       GNUNET_BLOCK_TYPE_FS_ONDEMAND,
 556                                       p->bo.content_priority,
 557                                       p->bo.anonymity_level,
 558                                       p->bo.replication_level,
 559                                       p->bo.expiration_time,
 560                                       -2, 1,
 561                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 562                                       &ds_put_cont,
 563                                       pc);
 564       return;
 565     }
 566 #if DEBUG_PUBLISH
 567   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 568               "Publishing block `%s' for offset %llu with size %u\n",
 569               GNUNET_h2s (&chk->query),
 570               (unsigned long long) offset,
 571               (unsigned int) block_size);
 572 #endif
 573   GNUNET_assert (pc->qre == NULL);
 574   pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 575                                   (p->is_directory) ? 0 : pc->rid,
 576                                   &chk->query,
 577                                   block_size,
 578                                   block,
 579                                   type,
 580                                   p->bo.content_priority,
 581                                   p->bo.anonymity_level,
 582                                   p->bo.replication_level,
 583                                   p->bo.expiration_time,
 584                                   -2, 1,
 585                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 586                                   &ds_put_cont,
 587                                   pc);
 588 }
 589
 590
 591 /**
 592  * Function called with information about our
 593  * progress in computing the tree encoding.
 594  *
 595  * @param cls closure
 596  * @param offset where are we in the file
 597  * @param pt_block plaintext of the currently processed block
 598  * @param pt_size size of pt_block
 599  * @param depth depth of the block in the tree, 0 for DBLOCK
 600  */
 601 static void
 602 progress_proc (void *cls,
 603                uint64_t offset,
 604                const void *pt_block,
 605                size_t pt_size,
 606                unsigned int depth)
 607 {
 608   struct GNUNET_FS_PublishContext *pc = cls;
 609   struct GNUNET_FS_FileInformation *p;
 610   struct GNUNET_FS_ProgressInfo pi;
 611
 612   p = pc->fi_pos;
 613   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 614   pi.value.publish.specifics.progress.data = pt_block;
 615   pi.value.publish.specifics.progress.offset = offset;
 616   pi.value.publish.specifics.progress.data_len = pt_size;
 617   pi.value.publish.specifics.progress.depth = depth;
 618   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset);
 619 }
 620
 621
 622 /**
 623  * We are uploading a file or directory; load (if necessary) the next
 624  * block into memory, encrypt it and send it to the FS service.  Then
 625  * continue with the main task.
 626  *
 627  * @param pc overall upload data
 628  */
 629 static void
 630 publish_content (struct GNUNET_FS_PublishContext *pc)
 631 {
 632   struct GNUNET_FS_FileInformation *p;
 633   char *emsg;
 634   struct GNUNET_FS_DirectoryBuilder *db;
 635   struct GNUNET_FS_FileInformation *dirpos;
 636   void *raw_data;
 637   uint64_t size;
 638
 639   p = pc->fi_pos;
 640   GNUNET_assert (p != NULL);
 641   if (NULL == p->te)
 642     {
 643       if (p->is_directory)
 644         {
 645 #if DEBUG_PUBLISH
 646           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 647                       "Creating directory\n");
 648 #endif
 649           db = GNUNET_FS_directory_builder_create (p->meta);
 650           dirpos = p->data.dir.entries;
 651           while (NULL != dirpos)
 652             {
 653               if (dirpos->is_directory)
 654                 {
 655                   raw_data = dirpos->data.dir.dir_data;
 656                   dirpos->data.dir.dir_data = NULL;
 657                 }
 658               else
 659                 {
 660                   raw_data = NULL;
 661                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 662                        (dirpos->data.file.file_size > 0) )
 663                     {
 664                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 665                       emsg = NULL;
 666                       if (dirpos->data.file.file_size !=
 667                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 668                                                     0,
 669                                                     dirpos->data.file.file_size,
 670                                                     raw_data,
 671                                                     &emsg))
 672                         {
 673                           GNUNET_free_non_null (emsg);
 674                           GNUNET_free (raw_data);
 675                           raw_data = NULL;
 676                         }
 677                     }
 678                 }
 679               GNUNET_FS_directory_builder_add (db,
 680                                                dirpos->chk_uri,
 681                                                dirpos->meta,
 682                                                raw_data);
 683               GNUNET_free_non_null (raw_data);
 684               dirpos = dirpos->next;
 685             }
 686           GNUNET_free_non_null (p->data.dir.dir_data);
 687           p->data.dir.dir_data = NULL;
 688           p->data.dir.dir_size = 0;
 689           GNUNET_FS_directory_builder_finish (db,
 690                                               &p->data.dir.dir_size,
 691                                               &p->data.dir.dir_data);
 692           GNUNET_FS_file_information_sync_ (p);
 693         }
 694       size = (p->is_directory)
 695         ? p->data.dir.dir_size
 696         : p->data.file.file_size;
 697 #if DEBUG_PUBLISH
 698       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 699                   "Creating tree encoder\n");
 700 #endif
 701       p->te = GNUNET_FS_tree_encoder_create (pc->h,
 702                                              size,
 703                                              pc,
 704                                              &block_reader,
 705                                              &block_proc,
 706                                              &progress_proc,
 707                                              &encode_cont);
 708
 709     }
 710 #if DEBUG_PUBLISH
 711   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 712               "Processing next block from tree\n");
 713 #endif
 714   GNUNET_FS_tree_encoder_next (p->te);
 715 }
 716
 717
 718 /**
 719  * Process the response (or lack thereof) from
 720  * the "fs" service to our 'start index' request.
 721  *
 722  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 723  * @param msg the response we got
 724  */
 725 static void
 726 process_index_start_response (void *cls,
 727                               const struct GNUNET_MessageHeader *msg)
 728 {
 729   struct GNUNET_FS_PublishContext *pc = cls;
 730   struct GNUNET_FS_FileInformation *p;
 731   const char *emsg;
 732   uint16_t msize;
 733
 734   GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 735   pc->client = NULL;
 736   p = pc->fi_pos;
 737   if (msg == NULL)
 738     {
 739       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 740                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 741                   p->filename,
 742                   _("timeout on index-start request to `fs' service"));
 743       p->data.file.do_index = GNUNET_NO;
 744       GNUNET_FS_file_information_sync_ (p);
 745       publish_content (pc);
 746       return;
 747     }
 748   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 749     {
 750       msize = ntohs (msg->size);
 751       emsg = (const char *) &msg[1];
 752       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 753            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 754         emsg = gettext_noop ("unknown error");
 755       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 756                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 757                   p->filename,
 758                   gettext (emsg));
 759       p->data.file.do_index = GNUNET_NO;
 760       GNUNET_FS_file_information_sync_ (p);
 761       publish_content (pc);
 762       return;
 763     }
 764   p->data.file.index_start_confirmed = GNUNET_YES;
 765   /* success! continue with indexing */
 766   GNUNET_FS_file_information_sync_ (p);
 767   publish_content (pc);
 768 }
 769
 770
 771 /**
 772  * Function called once the hash computation over an
 773  * indexed file has completed.
 774  *
 775  * @param cls closure, our publishing context
 776  * @param res resulting hash, NULL on error
 777  */
 778 static void
 779 hash_for_index_cb (void *cls,
 780                    const GNUNET_HashCode *
 781                    res)
 782 {
 783   struct GNUNET_FS_PublishContext *pc = cls;
 784   struct GNUNET_FS_FileInformation *p;
 785   struct IndexStartMessage *ism;
 786   size_t slen;
 787   struct GNUNET_CLIENT_Connection *client;
 788   uint64_t dev;
 789   uint64_t ino;
 790   char *fn;
 791
 792   pc->fhc = NULL;
 793   p = pc->fi_pos;
 794   if (NULL == res)
 795     {
 796       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 797                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 798                   p->filename,
 799                   _("failed to compute hash"));
 800       p->data.file.do_index = GNUNET_NO;
 801       GNUNET_FS_file_information_sync_ (p);
 802       publish_content (pc);
 803       return;
 804     }
 805   if (GNUNET_YES == p->data.file.index_start_confirmed)
 806     {
 807       publish_content (pc);
 808       return;
 809     }
 810   fn = GNUNET_STRINGS_filename_expand (p->filename);
 811   GNUNET_assert (fn != NULL);
 812   slen = strlen (fn) + 1;
 813   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 814     {
 815       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 816                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 817                   fn,
 818                   _("filename too long"));
 819       GNUNET_free (fn);
 820       p->data.file.do_index = GNUNET_NO;
 821       GNUNET_FS_file_information_sync_ (p);
 822       publish_content (pc);
 823       return;
 824     }
 825 #if DEBUG_PUBLISH
 826   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 827               "Hash of indexed file `%s' is `%s'\n",
 828               p->filename,
 829               GNUNET_h2s (res));
 830 #endif
 831   if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
 832     {
 833       p->data.file.file_id = *res;
 834       p->data.file.have_hash = GNUNET_YES;
 835       p->data.file.index_start_confirmed = GNUNET_YES;
 836       GNUNET_FS_file_information_sync_ (p);
 837       publish_content (pc);
 838       GNUNET_free (fn);
 839       return;
 840     }
 841   client = GNUNET_CLIENT_connect ("fs",
 842                                   pc->h->cfg);
 843   if (NULL == client)
 844     {
 845       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 846                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 847                   p->filename,
 848                   _("could not connect to `fs' service"));
 849       p->data.file.do_index = GNUNET_NO;
 850       publish_content (pc);
 851       GNUNET_free (fn);
 852       return;
 853     }
 854   if (p->data.file.have_hash != GNUNET_YES)
 855     {
 856       p->data.file.file_id = *res;
 857       p->data.file.have_hash = GNUNET_YES;
 858       GNUNET_FS_file_information_sync_ (p);
 859     }
 860   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 861                        slen);
 862   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 863                            slen);
 864   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 865   if (GNUNET_OK ==
 866       GNUNET_DISK_file_get_identifiers (p->filename,
 867                                         &dev,
 868                                         &ino))
 869     {
 870       ism->device = GNUNET_htonll (dev);
 871       ism->inode = GNUNET_htonll(ino);
 872     }
 873 #if DEBUG_PUBLISH
 874   else
 875     {
 876       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 877                   _("Failed to get file identifiers for `%s'\n"),
 878                   p->filename);
 879     }
 880 #endif
 881   ism->file_id = *res;
 882   memcpy (&ism[1],
 883           fn,
 884           slen);
 885   GNUNET_free (fn);
 886   pc->client = client;
 887   GNUNET_break (GNUNET_YES ==
 888                 GNUNET_CLIENT_transmit_and_get_response (client,
 889                                                          &ism->header,
 890                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 891                                                          GNUNET_YES,
 892                                                          &process_index_start_response,
 893                                                          pc));
 894   GNUNET_free (ism);
 895 }
 896
 897
 898 /**
 899  * Main function that performs the upload.
 900  *
 901  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 902  * @param tc task context
 903  */
 904 void
 905 GNUNET_FS_publish_main_ (void *cls,
 906                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 907 {
 908   struct GNUNET_FS_PublishContext *pc = cls;
 909   struct GNUNET_FS_ProgressInfo pi;
 910   struct GNUNET_FS_FileInformation *p;
 911   struct GNUNET_FS_Uri *loc;
 912   char *fn;
 913
 914   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 915   p = pc->fi_pos;
 916   if (NULL == p)
 917     {
 918 #if DEBUG_PUBLISH
 919       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 920                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 921 #endif
 922       /* upload of entire hierarchy complete,
 923          publish namespace entries */
 924       GNUNET_FS_publish_sync_ (pc);
 925       publish_sblock (pc);
 926       return;
 927     }
 928   /* find starting position */
 929   while ( (p->is_directory) &&
 930           (NULL != p->data.dir.entries) &&
 931           (NULL == p->emsg) &&
 932           (NULL == p->data.dir.entries->chk_uri) )
 933     {
 934       p = p->data.dir.entries;
 935       pc->fi_pos = p;
 936       GNUNET_FS_publish_sync_ (pc);
 937     }
 938   /* abort on error */
 939   if (NULL != p->emsg)
 940     {
 941 #if DEBUG_PUBLISH
 942       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 943                   "Error uploading: %s\n",
 944                   p->emsg);
 945 #endif
 946       /* error with current file, abort all
 947          related files as well! */
 948       while (NULL != p->dir)
 949         {
 950           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 951                                                        EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
 952           p = p->dir;
 953           if (fn != NULL)
 954             {
 955               GNUNET_asprintf (&p->emsg,
 956                                _("Recursive upload failed at `%s': %s"),
 957                                fn,
 958                                p->emsg);
 959               GNUNET_free (fn);
 960             }
 961           else
 962             {
 963               GNUNET_asprintf (&p->emsg,
 964                                _("Recursive upload failed: %s"),
 965                                p->emsg);
 966             }
 967           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 968           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 969           pi.value.publish.specifics.error.message = p->emsg;
 970           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 971         }
 972       pc->all_done = GNUNET_YES;
 973       GNUNET_FS_publish_sync_ (pc);
 974       return;
 975     }
 976   /* handle completion */
 977   if (NULL != p->chk_uri)
 978     {
 979 #if DEBUG_PUBLISH
 980       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 981                   "File upload complete, now publishing KSK blocks.\n");
 982 #endif
 983       if (0 == p->bo.anonymity_level)
 984         {
 985           /* zero anonymity, box CHK URI in LOC URI */
 986           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
 987                                           pc->h->cfg,
 988                                           p->bo.expiration_time);
 989           GNUNET_FS_uri_destroy (p->chk_uri);
 990           p->chk_uri = loc;
 991         }
 992       GNUNET_FS_publish_sync_ (pc);
 993       /* upload of "p" complete, publish KBlocks! */
 994       if (p->keywords != NULL)
 995         {
 996           GNUNET_FS_publish_ksk (pc->h,
 997                                  p->keywords,
 998                                  p->meta,
 999                                  p->chk_uri,
1000                                  &p->bo,
1001                                  pc->options,
1002                                  &publish_kblocks_cont,
1003                                  pc);
1004         }
1005       else
1006         {
1007           publish_kblocks_cont (pc,
1008                                 p->chk_uri,
1009                                 NULL);
1010         }
1011       return;
1012     }
1013   if ( (!p->is_directory) &&
1014        (p->data.file.do_index) )
1015     {
1016       if (NULL == p->filename)
1017         {
1018           p->data.file.do_index = GNUNET_NO;
1019           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1020                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1021                       "<no-name>",
1022                       _("needs to be an actual file"));
1023           GNUNET_FS_file_information_sync_ (p);
1024           publish_content (pc);
1025           return;
1026         }
1027       if (p->data.file.have_hash)
1028         {
1029           hash_for_index_cb (pc,
1030                              &p->data.file.file_id);
1031         }
1032       else
1033         {
1034           p->start_time = GNUNET_TIME_absolute_get ();
1035           pc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
1036                                              p->filename,
1037                                              HASHING_BLOCKSIZE,
1038                                              &hash_for_index_cb,
1039                                              pc);
1040         }
1041       return;
1042     }
1043   publish_content (pc);
1044 }
1045
1046
1047 /**
1048  * Signal the FS's progress function that we are starting
1049  * an upload.
1050  *
1051  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1052  * @param fi the entry in the publish-structure
1053  * @param length length of the file or directory
1054  * @param meta metadata for the file or directory (can be modified)
1055  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1056  * @param bo block options
1057  * @param do_index should we index?
1058  * @param client_info pointer to client context set upon creation (can be modified)
1059  * @return GNUNET_OK to continue (always)
1060  */
1061 static int
1062 fip_signal_start(void *cls,
1063                  struct GNUNET_FS_FileInformation *fi,
1064                  uint64_t length,
1065                  struct GNUNET_CONTAINER_MetaData *meta,
1066                  struct GNUNET_FS_Uri **uri,
1067                  struct GNUNET_FS_BlockOptions *bo,
1068                  int *do_index,
1069                  void **client_info)
1070 {
1071   struct GNUNET_FS_PublishContext *pc = cls;
1072   struct GNUNET_FS_ProgressInfo pi;
1073   unsigned int kc;
1074   uint64_t left;
1075
1076 #if DEBUG_PUBLISH
1077   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1078               "Starting publish operation\n");
1079 #endif
1080   if (*do_index)
1081     {
1082       /* space for on-demand blocks */
1083       pc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1084     }
1085   else
1086     {
1087       /* space for DBlocks */
1088       pc->reserve_space += length;
1089     }
1090   /* entries for IBlocks and DBlocks, space for IBlocks */
1091   left = length;
1092   while (1)
1093     {
1094       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1095       pc->reserve_entries += left;
1096       if (left <= 1)
1097         break;
1098       left = left * sizeof (struct ContentHashKey);
1099       pc->reserve_space += left;
1100     }
1101   pc->reserve_entries++;
1102   /* entries and space for keywords */
1103   if (NULL != *uri)
1104     {
1105       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1106       pc->reserve_entries += kc;
1107       pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1108     }
1109   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1110   *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
1111   GNUNET_FS_file_information_sync_ (fi);
1112   return GNUNET_OK;
1113 }
1114
1115
1116 /**
1117  * Signal the FS's progress function that we are suspending
1118  * an upload.
1119  *
1120  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1121  * @param fi the entry in the publish-structure
1122  * @param length length of the file or directory
1123  * @param meta metadata for the file or directory (can be modified)
1124  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1125  * @param bo block options
1126  * @param do_index should we index?
1127  * @param client_info pointer to client context set upon creation (can be modified)
1128  * @return GNUNET_OK to continue (always)
1129  */
1130 static int
1131 fip_signal_suspend(void *cls,
1132                    struct GNUNET_FS_FileInformation *fi,
1133                    uint64_t length,
1134                    struct GNUNET_CONTAINER_MetaData *meta,
1135                    struct GNUNET_FS_Uri **uri,
1136                    struct GNUNET_FS_BlockOptions *bo,
1137                    int *do_index,
1138                    void **client_info)
1139 {
1140   struct GNUNET_FS_PublishContext*pc = cls;
1141   struct GNUNET_FS_ProgressInfo pi;
1142   uint64_t off;
1143
1144 #if DEBUG_PUBLISH
1145   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1146               "Suspending publish operation\n");
1147 #endif
1148   GNUNET_free_non_null (fi->serialization);
1149   fi->serialization = NULL;
1150   off = (fi->chk_uri == NULL) ? 0 : length;
1151   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1152   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1153   *client_info = NULL;
1154   if (NULL != pc->qre)
1155     {
1156       GNUNET_DATASTORE_cancel (pc->qre);
1157       pc->qre = NULL;
1158     }
1159   if (NULL != pc->dsh)
1160     {
1161       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1162       pc->dsh = NULL;
1163     }
1164   pc->rid = 0;
1165   return GNUNET_OK;
1166 }
1167
1168
1169 /**
1170  * Create SUSPEND event for the given publish operation
1171  * and then clean up our state (without stop signal).
1172  *
1173  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1174  */
1175 void
1176 GNUNET_FS_publish_signal_suspend_ (void *cls)
1177 {
1178   struct GNUNET_FS_PublishContext *pc = cls;
1179
1180   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1181     {
1182       GNUNET_SCHEDULER_cancel (pc->upload_task);
1183       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1184     }
1185   GNUNET_FS_file_information_inspect (pc->fi,
1186                                       &fip_signal_suspend,
1187                                       pc);
1188   GNUNET_FS_end_top (pc->h, pc->top);
1189   pc->top = NULL;
1190   publish_cleanup (pc, NULL);
1191 }
1192
1193
1194 /**
1195  * We have gotten a reply for our space reservation request.
1196  * Either fail (insufficient space) or start publishing for good.
1197  *
1198  * @param cls the 'struct GNUNET_FS_PublishContext*'
1199  * @param success positive reservation ID on success
1200  * @param msg error message on error, otherwise NULL
1201  */
1202 static void
1203 finish_reserve (void *cls,
1204                 int success,
1205                 const char *msg)
1206 {
1207   struct GNUNET_FS_PublishContext *pc = cls;
1208
1209   pc->qre = NULL;
1210 #if DEBUG_PUBLISH
1211   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1212               "Reservation complete (%d)!\n",
1213               success);
1214 #endif
1215   if ( (msg != NULL) ||
1216        (success <= 0) )
1217     {
1218       GNUNET_asprintf (&pc->fi->emsg,
1219                        _("Insufficient space for publishing: %s"),
1220                        msg);
1221       signal_publish_error (pc->fi,
1222                             pc,
1223                             pc->fi->emsg);
1224       return;
1225     }
1226   pc->rid = success;
1227   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
1228   pc->upload_task
1229     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1230                                           &GNUNET_FS_publish_main_,
1231                                           pc);
1232 }
1233
1234
1235 /**
1236  * Publish a file or directory.
1237  *
1238  * @param h handle to the file sharing subsystem
1239  * @param fi information about the file or directory structure to publish
1240  * @param namespace namespace to publish the file in, NULL for no namespace
1241  * @param nid identifier to use for the publishd content in the namespace
1242  *        (can be NULL, must be NULL if namespace is NULL)
1243  * @param nuid update-identifier that will be used for future updates
1244  *        (can be NULL, must be NULL if namespace or nid is NULL)
1245  * @param options options for the publication
1246  * @return context that can be used to control the publish operation
1247  */
1248 struct GNUNET_FS_PublishContext *
1249 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1250                          struct GNUNET_FS_FileInformation *fi,
1251                          struct GNUNET_FS_Namespace *namespace,
1252                          const char *nid,
1253                          const char *nuid,
1254                          enum GNUNET_FS_PublishOptions options)
1255 {
1256   struct GNUNET_FS_PublishContext *ret;
1257   struct GNUNET_DATASTORE_Handle *dsh;
1258
1259   GNUNET_assert (NULL != h);
1260   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1261     {
1262       dsh = GNUNET_DATASTORE_connect (h->cfg);
1263       if (NULL == dsh)
1264         return NULL;
1265     }
1266   else
1267     {
1268       dsh = NULL;
1269     }
1270   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1271   ret->dsh = dsh;
1272   ret->h = h;
1273   ret->fi = fi;
1274   ret->namespace = namespace;
1275   ret->options = options;
1276   if (namespace != NULL)
1277     {
1278       namespace->rc++;
1279       GNUNET_assert (NULL != nid);
1280       ret->nid = GNUNET_strdup (nid);
1281       if (NULL != nuid)
1282         ret->nuid = GNUNET_strdup (nuid);
1283     }
1284   /* signal start */
1285   GNUNET_FS_file_information_inspect (ret->fi,
1286                                       &fip_signal_start,
1287                                       ret);
1288   ret->fi_pos = ret->fi;
1289   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1290   GNUNET_FS_publish_sync_ (ret);
1291   if (NULL != ret->dsh)
1292     {
1293       GNUNET_assert (NULL == ret->qre);
1294       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1295                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1296                   (unsigned int) ret->reserve_entries,
1297                   (unsigned long long) ret->reserve_space);
1298       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1299                                            ret->reserve_space,
1300                                            ret->reserve_entries,
1301                                            UINT_MAX,
1302                                            UINT_MAX,
1303                                            GNUNET_TIME_UNIT_FOREVER_REL,
1304                                            &finish_reserve,
1305                                            ret);
1306     }
1307   else
1308     {
1309       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == ret->upload_task);
1310       ret->upload_task
1311         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1312                                               &GNUNET_FS_publish_main_,
1313                                               ret);
1314     }
1315   return ret;
1316 }
1317
1318
1319 /**
1320  * Signal the FS's progress function that we are stopping
1321  * an upload.
1322  *
1323  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1324  * @param fi the entry in the publish-structure
1325  * @param length length of the file or directory
1326  * @param meta metadata for the file or directory (can be modified)
1327  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1328  * @param bo block options (can be modified)
1329  * @param do_index should we index?
1330  * @param client_info pointer to client context set upon creation (can be modified)
1331  * @return GNUNET_OK to continue (always)
1332  */
1333 static int
1334 fip_signal_stop(void *cls,
1335                 struct GNUNET_FS_FileInformation *fi,
1336                 uint64_t length,
1337                 struct GNUNET_CONTAINER_MetaData *meta,
1338                 struct GNUNET_FS_Uri **uri,
1339                 struct GNUNET_FS_BlockOptions *bo,
1340                 int *do_index,
1341                 void **client_info)
1342 {
1343   struct GNUNET_FS_PublishContext*pc = cls;
1344   struct GNUNET_FS_ProgressInfo pi;
1345   uint64_t off;
1346
1347   if (fi->serialization != NULL)
1348     {
1349       GNUNET_FS_remove_sync_file_ (pc->h,
1350                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1351                                    fi->serialization);
1352       GNUNET_free (fi->serialization);
1353       fi->serialization = NULL;
1354     }
1355   off = (fi->chk_uri == NULL) ? 0 : length;
1356   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1357   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1358   *client_info = NULL;
1359   return GNUNET_OK;
1360 }
1361
1362
1363 /**
1364  * Stop an upload.  Will abort incomplete uploads (but
1365  * not remove blocks that have already been publishd) or
1366  * simply clean up the state for completed uploads.
1367  * Must NOT be called from within the event callback!
1368  *
1369  * @param pc context for the upload to stop
1370  */
1371 void
1372 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1373 {
1374 #if DEBUG_PUBLISH
1375   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1376               "Publish stop called\n");
1377 #endif
1378   GNUNET_FS_end_top (pc->h, pc->top);
1379   if (NULL != pc->qre)
1380     {
1381       GNUNET_DATASTORE_cancel (pc->qre);
1382       pc->qre = NULL;
1383     }
1384   if (NULL != pc->dsh)
1385     {
1386       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1387       pc->dsh = NULL;
1388     }
1389   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1390     {
1391       GNUNET_SCHEDULER_cancel (pc->upload_task);
1392       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1393     }
1394   if (pc->serialization != NULL)
1395     {
1396       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1397       GNUNET_free (pc->serialization);
1398       pc->serialization = NULL;
1399     }
1400   GNUNET_FS_file_information_inspect (pc->fi,
1401                                       &fip_signal_stop,
1402                                       pc);
1403   if (GNUNET_YES == pc->in_network_wait)
1404     {
1405       pc->in_network_wait = GNUNET_SYSERR;
1406       return;
1407     }
1408   publish_cleanup (pc, NULL);
1409 }
1410
1411
1412 /**
1413  * Context for the KSK publication.
1414  */
1415 struct PublishKskContext
1416 {
1417
1418   /**
1419    * Keywords to use.
1420    */
1421   struct GNUNET_FS_Uri *ksk_uri;
1422
1423   /**
1424    * Global FS context.
1425    */
1426   struct GNUNET_FS_Handle *h;
1427
1428   /**
1429    * The master block that we are sending
1430    * (in plaintext), has "mdsize+slen" more
1431    * bytes than the struct would suggest.
1432    */
1433   struct KBlock *kb;
1434
1435   /**
1436    * Buffer of the same size as "kb" for
1437    * the encrypted version.
1438    */
1439   struct KBlock *cpy;
1440
1441   /**
1442    * Handle to the datastore, NULL if we are just
1443    * simulating.
1444    */
1445   struct GNUNET_DATASTORE_Handle *dsh;
1446
1447   /**
1448    * Handle to datastore PUT request.
1449    */
1450   struct GNUNET_DATASTORE_QueueEntry *qre;
1451
1452   /**
1453    * Function to call once we're done.
1454    */
1455   GNUNET_FS_PublishContinuation cont;
1456
1457   /**
1458    * Closure for cont.
1459    */
1460   void *cont_cls;
1461
1462   /**
1463    * When should the KBlocks expire?
1464    */
1465   struct GNUNET_FS_BlockOptions bo;
1466
1467   /**
1468    * Size of the serialized metadata.
1469    */
1470   ssize_t mdsize;
1471
1472   /**
1473    * Size of the (CHK) URI as a string.
1474    */
1475   size_t slen;
1476
1477   /**
1478    * Keyword that we are currently processing.
1479    */
1480   unsigned int i;
1481
1482 };
1483
1484
1485 /**
1486  * Continuation of "GNUNET_FS_publish_ksk" that performs
1487  * the actual publishing operation (iterating over all
1488  * of the keywords).
1489  *
1490  * @param cls closure of type "struct PublishKskContext*"
1491  * @param tc unused
1492  */
1493 static void
1494 publish_ksk_cont (void *cls,
1495                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1496
1497
1498 /**
1499  * Function called by the datastore API with
1500  * the result from the PUT request.
1501  *
1502  * @param cls closure of type "struct PublishKskContext*"
1503  * @param success GNUNET_OK on success
1504  * @param msg error message (or NULL)
1505  */
1506 static void
1507 kb_put_cont (void *cls,
1508              int success,
1509              const char *msg)
1510 {
1511   struct PublishKskContext *pkc = cls;
1512
1513   pkc->qre = NULL;
1514   if (GNUNET_OK != success)
1515     {
1516 #if DEBUG_PUBLISH
1517       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1518                   "KB PUT operation complete\n");
1519 #endif
1520       if (NULL != pkc->dsh)
1521         {
1522           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1523           pkc->dsh = NULL;
1524         }
1525       GNUNET_free (pkc->cpy);
1526       GNUNET_free (pkc->kb);
1527       pkc->cont (pkc->cont_cls,
1528                  NULL,
1529                  msg);
1530       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1531       GNUNET_free (pkc);
1532       return;
1533     }
1534   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1535                                      pkc,
1536                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1537 }
1538
1539
1540 /**
1541  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1542  * publishing operation (iterating over all of the keywords).
1543  *
1544  * @param cls closure of type "struct PublishKskContext*"
1545  * @param tc unused
1546  */
1547 static void
1548 publish_ksk_cont (void *cls,
1549                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1550 {
1551   struct PublishKskContext *pkc = cls;
1552   const char *keyword;
1553   GNUNET_HashCode key;
1554   GNUNET_HashCode query;
1555   struct GNUNET_CRYPTO_AesSessionKey skey;
1556   struct GNUNET_CRYPTO_AesInitializationVector iv;
1557   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1558
1559
1560   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1561        (NULL == pkc->dsh) )
1562     {
1563 #if DEBUG_PUBLISH
1564       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1565                   "KSK PUT operation complete\n");
1566 #endif
1567       if (NULL != pkc->dsh)
1568         {
1569           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1570           pkc->dsh = NULL;
1571         }
1572       GNUNET_free (pkc->cpy);
1573       GNUNET_free (pkc->kb);
1574       pkc->cont (pkc->cont_cls,
1575                  pkc->ksk_uri,
1576                  NULL);
1577       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1578       GNUNET_free (pkc);
1579       return;
1580     }
1581   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1582 #if DEBUG_PUBLISH
1583   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1584               "Publishing under keyword `%s'\n",
1585               keyword);
1586 #endif
1587   /* first character of keyword indicates if it is
1588      mandatory or not -- ignore for hashing */
1589   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1590   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1591   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1592                              pkc->slen + pkc->mdsize,
1593                              &skey,
1594                              &iv,
1595                              &pkc->cpy[1]);
1596   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1597   GNUNET_assert (NULL != pk);
1598   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1599   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1600                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1601                       &query);
1602   GNUNET_assert (GNUNET_OK ==
1603                  GNUNET_CRYPTO_rsa_sign (pk,
1604                                          &pkc->cpy->purpose,
1605                                          &pkc->cpy->signature));
1606   GNUNET_CRYPTO_rsa_key_free (pk);
1607   pkc->qre = GNUNET_DATASTORE_put (pkc->dsh,
1608                                    0,
1609                                    &query,
1610                                    pkc->mdsize +
1611                                    sizeof (struct KBlock) +
1612                                    pkc->slen,
1613                                    pkc->cpy,
1614                                    GNUNET_BLOCK_TYPE_FS_KBLOCK,
1615                                    pkc->bo.content_priority,
1616                                    pkc->bo.anonymity_level,
1617                                    pkc->bo.replication_level,
1618                                    pkc->bo.expiration_time,
1619                                    -2, 1,
1620                                    GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1621                                    &kb_put_cont,
1622                                    pkc);
1623 }
1624
1625
1626 /**
1627  * Publish a CHK under various keywords on GNUnet.
1628  *
1629  * @param h handle to the file sharing subsystem
1630  * @param ksk_uri keywords to use
1631  * @param meta metadata to use
1632  * @param uri URI to refer to in the KBlock
1633  * @param bo per-block options
1634  * @param options publication options
1635  * @param cont continuation
1636  * @param cont_cls closure for cont
1637  */
1638 void
1639 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1640                        const struct GNUNET_FS_Uri *ksk_uri,
1641                        const struct GNUNET_CONTAINER_MetaData *meta,
1642                        const struct GNUNET_FS_Uri *uri,
1643                        const struct GNUNET_FS_BlockOptions *bo,
1644                        enum GNUNET_FS_PublishOptions options,
1645                        GNUNET_FS_PublishContinuation cont,
1646                        void *cont_cls)
1647 {
1648   struct PublishKskContext *pkc;
1649   char *uris;
1650   size_t size;
1651   char *kbe;
1652   char *sptr;
1653
1654   GNUNET_assert (NULL != uri);
1655   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1656   pkc->h = h;
1657   pkc->bo = *bo;
1658   pkc->cont = cont;
1659   pkc->cont_cls = cont_cls;
1660   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1661     {
1662       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
1663       if (pkc->dsh == NULL)
1664         {
1665           cont (cont_cls, NULL, _("Could not connect to datastore."));
1666           GNUNET_free (pkc);
1667           return;
1668         }
1669     }
1670   if (meta == NULL)
1671     pkc->mdsize = 0;
1672   else
1673     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1674   GNUNET_assert (pkc->mdsize >= 0);
1675   uris = GNUNET_FS_uri_to_string (uri);
1676   pkc->slen = strlen (uris) + 1;
1677   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1678   if (size > MAX_KBLOCK_SIZE)
1679     {
1680       size = MAX_KBLOCK_SIZE;
1681       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1682     }
1683   pkc->kb = GNUNET_malloc (size);
1684   kbe = (char *) &pkc->kb[1];
1685   memcpy (kbe, uris, pkc->slen);
1686   GNUNET_free (uris);
1687   sptr = &kbe[pkc->slen];
1688   if (meta != NULL)
1689     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1690                                                         &sptr,
1691                                                         pkc->mdsize,
1692                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1693   if (pkc->mdsize == -1)
1694     {
1695       GNUNET_break (0);
1696       GNUNET_free (pkc->kb);
1697       if (pkc->dsh != NULL)
1698         {
1699           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1700           pkc->dsh = NULL;
1701         }
1702       cont (cont_cls, NULL, _("Internal error."));
1703       GNUNET_free (pkc);
1704       return;
1705     }
1706   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1707
1708   pkc->cpy = GNUNET_malloc (size);
1709   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1710                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1711                                   pkc->mdsize +
1712                                   pkc->slen);
1713   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1714   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1715   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1716                                      pkc,
1717                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1718 }
1719
1720
1721 /* end of fs_publish.c */