src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see https://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Fill in all of the generic fields for
  42  * a publish event and call the callback.
  43  *
  44  * @param pi structure to fill in
  45  * @param pc overall publishing context
  46  * @param p file information for the file being published
  47  * @param offset where in the file are we so far
  48  * @return value returned from callback
  49  */
  50 void *
  51 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  52                                 struct GNUNET_FS_PublishContext *pc,
  53                                 const struct GNUNET_FS_FileInformation *p,
  54                                 uint64_t offset)
  55 {
  56   pi->value.publish.pc = pc;
  57   pi->value.publish.fi = p;
  58   pi->value.publish.cctx
  59     = p->client_info;
  60   pi->value.publish.pctx
  61     = (NULL == p->dir) ? NULL : p->dir->client_info;
  62   pi->value.publish.filename = p->filename;
  63   pi->value.publish.size
  64     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  65   pi->value.publish.eta
  66     = GNUNET_TIME_calculate_eta (p->start_time,
  67                                  offset,
  68                                  pi->value.publish.size);
  69   pi->value.publish.completed = offset;
  70   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  71   pi->value.publish.anonymity = p->bo.anonymity_level;
  72   return pc->h->upcb (pc->h->upcb_cls,
  73                       pi);
  74 }
  75
  76
  77 /**
  78  * Cleanup the publish context, we're done with it.
  79  *
  80  * @param cls struct to clean up after
  81  * @param tc scheduler context
  82  */
  83 static void
  84 publish_cleanup (void *cls,
  85                  const struct GNUNET_SCHEDULER_TaskContext *tc)
  86 {
  87   struct GNUNET_FS_PublishContext *pc = cls;
  88
  89 #if DEBUG_PUBLISH
  90   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
  91               "Cleaning up publish context (done!)\n");
  92 #endif
  93   if (pc->fhc != NULL)
  94     {
  95       GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
  96       pc->fhc = NULL;
  97     }
  98   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
  99   if (pc->namespace != NULL)
 100     {
 101       GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 102       pc->namespace = NULL;
 103     }
 104   GNUNET_free_non_null (pc->nid);
 105   GNUNET_free_non_null (pc->nuid);
 106   GNUNET_free_non_null (pc->serialization);
 107   if (pc->dsh != NULL)
 108     {
 109       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 110       pc->dsh = NULL;
 111     }
 112   if (pc->client != NULL)
 113     {
 114       GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 115       pc->client = NULL;
 116     }
 117   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 118   GNUNET_free (pc);
 119 }
 120
 121
 122 /**
 123  * Function called by the datastore API with
 124  * the result from the PUT request.
 125  *
 126  * @param cls the 'struct GNUNET_FS_PublishContext'
 127  * @param success GNUNET_OK on success
 128  * @param msg error message (or NULL)
 129  */
 130 static void
 131 ds_put_cont (void *cls,
 132              int success,
 133              const char *msg)
 134 {
 135   struct GNUNET_FS_PublishContext *pc = cls;
 136   struct GNUNET_FS_ProgressInfo pi;
 137
 138   pc->qre = NULL;
 139   if (GNUNET_SYSERR == pc->in_network_wait)
 140     {
 141       /* we were aborted in the meantime, finish shutdown! */
 142       GNUNET_SCHEDULER_add_continuation (&publish_cleanup,
 143                                          pc,
 144                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 145       return;
 146     }
 147   GNUNET_assert (GNUNET_YES == pc->in_network_wait);
 148   pc->in_network_wait = GNUNET_NO;
 149   if (GNUNET_SYSERR == success)
 150     {
 151       GNUNET_asprintf (&pc->fi_pos->emsg,
 152                        _("Publishing failed: %s"),
 153                        msg);
 154       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 155       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 156       pi.value.publish.specifics.error.message = pc->fi_pos->emsg;
 157       pc->fi_pos->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0);
 158       if ( (pc->fi_pos->is_directory == GNUNET_NO) &&
 159            (pc->fi_pos->filename != NULL) &&
 160            (pc->fi_pos->data.file.do_index == GNUNET_YES) )
 161         {
 162           /* run unindex to clean up */
 163           GNUNET_FS_unindex_start (pc->h,
 164                                    pc->fi_pos->filename,
 165                                    NULL);
 166         }
 167     }
 168   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 169   pc->upload_task
 170     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 171                                           &GNUNET_FS_publish_main_,
 172                                           pc);
 173 }
 174
 175
 176 /**
 177  * Generate the callback that signals clients
 178  * that a file (or directory) has been completely
 179  * published.
 180  *
 181  * @param p the completed upload
 182  * @param pc context of the publication
 183  */
 184 static void
 185 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 186                            struct GNUNET_FS_PublishContext *pc)
 187 {
 188   struct GNUNET_FS_ProgressInfo pi;
 189
 190   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 191   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 192   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 193   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p,
 194                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 195 }
 196
 197
 198 /**
 199  * Generate the callback that signals clients
 200  * that a file (or directory) has encountered
 201  * a problem during publication.
 202  *
 203  * @param p the upload that had trouble
 204  * @param pc context of the publication
 205  * @param emsg error message
 206  */
 207 static void
 208 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 209                       struct GNUNET_FS_PublishContext *pc,
 210                       const char *emsg)
 211 {
 212   struct GNUNET_FS_ProgressInfo pi;
 213
 214   p->emsg = GNUNET_strdup (emsg);
 215   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 216   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 217   pi.value.publish.specifics.error.message =emsg;
 218   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 219   if ( (p->is_directory == GNUNET_NO) &&
 220        (p->filename != NULL) &&
 221        (p->data.file.do_index == GNUNET_YES) )
 222     {
 223       /* run unindex to clean up */
 224       GNUNET_FS_unindex_start (pc->h,
 225                                p->filename,
 226                                NULL);
 227     }
 228
 229 }
 230
 231
 232 /**
 233  * Datastore returns from reservation cancel request.
 234  *
 235  * @param cls the 'struct GNUNET_FS_PublishContext'
 236  * @param success success code (not used)
 237  * @param msg error message (typically NULL, not used)
 238  */
 239 static void
 240 finish_release_reserve (void *cls,
 241                         int success,
 242                         const char *msg)
 243 {
 244   struct GNUNET_FS_PublishContext *pc = cls;
 245
 246   pc->qre = NULL;
 247 #if DEBUG_PUBLISH
 248   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 249               "Releasing reserve done!\n");
 250 #endif
 251   signal_publish_completion (pc->fi, pc);
 252   pc->all_done = GNUNET_YES;
 253   GNUNET_FS_publish_sync_ (pc);
 254 }
 255
 256
 257 /**
 258  * We've finished publishing the SBlock as part of a larger upload.
 259  * Check the result and complete the larger upload.
 260  *
 261  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 262  * @param uri URI of the published SBlock
 263  * @param emsg NULL on success, otherwise error message
 264  */
 265 static void
 266 publish_sblocks_cont (void *cls,
 267                       const struct GNUNET_FS_Uri *uri,
 268                       const char *emsg)
 269 {
 270   struct GNUNET_FS_PublishContext *pc = cls;
 271   if (NULL != emsg)
 272     {
 273       signal_publish_error (pc->fi,
 274                             pc,
 275                             emsg);
 276       GNUNET_FS_publish_sync_ (pc);
 277       return;
 278     }
 279   GNUNET_assert (pc->qre == NULL);
 280   if ( (pc->dsh != NULL) &&
 281        (pc->rid != 0) )
 282     {
 283       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 284                                                   pc->rid,
 285                                                   UINT_MAX,
 286                                                   UINT_MAX,
 287                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 288                                                   &finish_release_reserve,
 289                                                   pc);
 290     }
 291   else
 292     {
 293       finish_release_reserve (pc, GNUNET_OK, NULL);
 294     }
 295 }
 296
 297
 298 /**
 299  * We are almost done publishing the structure,
 300  * add SBlocks (if needed).
 301  *
 302  * @param pc overall upload data
 303  */
 304 static void
 305 publish_sblock (struct GNUNET_FS_PublishContext *pc)
 306 {
 307   if (NULL != pc->namespace)
 308     GNUNET_FS_publish_sks (pc->h,
 309                            pc->namespace,
 310                            pc->nid,
 311                            pc->nuid,
 312                            pc->fi->meta,
 313                            pc->fi->chk_uri,
 314                            &pc->fi->bo,
 315                            pc->options,
 316                            &publish_sblocks_cont,
 317                            pc);
 318   else
 319     publish_sblocks_cont (pc, NULL, NULL);
 320 }
 321
 322
 323 /**
 324  * We've finished publishing a KBlock as part of a larger upload.
 325  * Check the result and continue the larger upload.
 326  *
 327  * @param cls the "struct GNUNET_FS_PublishContext*"
 328  *        of the larger upload
 329  * @param uri URI of the published blocks
 330  * @param emsg NULL on success, otherwise error message
 331  */
 332 static void
 333 publish_kblocks_cont (void *cls,
 334                       const struct GNUNET_FS_Uri *uri,
 335                       const char *emsg)
 336 {
 337   struct GNUNET_FS_PublishContext *pc = cls;
 338   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 339
 340   if (NULL != emsg)
 341     {
 342 #if DEBUG_PUBLISH
 343       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 344                   "Error uploading KSK blocks: %s\n",
 345                   emsg);
 346 #endif
 347       signal_publish_error (p, pc, emsg);
 348       GNUNET_FS_file_information_sync_ (p);
 349       GNUNET_FS_publish_sync_ (pc);
 350       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 351       pc->upload_task
 352         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 353                                               &GNUNET_FS_publish_main_,
 354                                               pc);
 355       return;
 356     }
 357 #if DEBUG_PUBLISH
 358   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 359               "KSK blocks published, moving on to next file\n");
 360 #endif
 361   if (NULL != p->dir)
 362     signal_publish_completion (p, pc);
 363   /* move on to next file */
 364   if (NULL != p->next)
 365     pc->fi_pos = p->next;
 366   else
 367     pc->fi_pos = p->dir;
 368   GNUNET_FS_publish_sync_ (pc);
 369   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 370   pc->upload_task
 371     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 372                                           &GNUNET_FS_publish_main_,
 373                                           pc);
 374 }
 375
 376
 377 /**
 378  * Function called by the tree encoder to obtain
 379  * a block of plaintext data (for the lowest level
 380  * of the tree).
 381  *
 382  * @param cls our publishing context
 383  * @param offset identifies which block to get
 384  * @param max (maximum) number of bytes to get; returning
 385  *        fewer will also cause errors
 386  * @param buf where to copy the plaintext buffer
 387  * @param emsg location to store an error message (on error)
 388  * @return number of bytes copied to buf, 0 on error
 389  */
 390 static size_t
 391 block_reader (void *cls,
 392               uint64_t offset,
 393               size_t max,
 394               void *buf,
 395               char **emsg)
 396 {
 397   struct GNUNET_FS_PublishContext *pc = cls;
 398   struct GNUNET_FS_FileInformation *p;
 399   size_t pt_size;
 400   const char *dd;
 401
 402   p = pc->fi_pos;
 403   if (p->is_directory)
 404     {
 405       pt_size = GNUNET_MIN(max,
 406                            p->data.dir.dir_size - offset);
 407       dd = p->data.dir.dir_data;
 408       memcpy (buf,
 409               &dd[offset],
 410               pt_size);
 411     }
 412   else
 413     {
 414       pt_size = GNUNET_MIN(max,
 415                            p->data.file.file_size - offset);
 416       if (pt_size == 0)
 417         return 0; /* calling reader with pt_size==0
 418                      might free buf, so don't! */
 419       if (pt_size !=
 420           p->data.file.reader (p->data.file.reader_cls,
 421                                offset,
 422                                pt_size,
 423                                buf,
 424                                emsg))
 425         return 0;
 426     }
 427   return pt_size;
 428 }
 429
 430
 431 /**
 432  * The tree encoder has finished processing a
 433  * file.   Call it's finish method and deal with
 434  * the final result.
 435  *
 436  * @param cls our publishing context
 437  * @param tc scheduler's task context (not used)
 438  */
 439 static void
 440 encode_cont (void *cls,
 441              const struct GNUNET_SCHEDULER_TaskContext *tc)
 442 {
 443   struct GNUNET_FS_PublishContext *pc = cls;
 444   struct GNUNET_FS_FileInformation *p;
 445   struct GNUNET_FS_ProgressInfo pi;
 446   char *emsg;
 447   uint64_t flen;
 448
 449   p = pc->fi_pos;
 450   GNUNET_FS_tree_encoder_finish (p->te,
 451                                  &p->chk_uri,
 452                                  &emsg);
 453   p->te = NULL;
 454   if (NULL != emsg)
 455     {
 456 #if DEBUG_PUBLISH
 457       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 458                   "Error during tree walk: %s\n",
 459                   emsg);
 460 #endif
 461       GNUNET_asprintf (&p->emsg,
 462                        _("Publishing failed: %s"),
 463                        emsg);
 464       GNUNET_free (emsg);
 465       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 466       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 467       pi.value.publish.specifics.error.message = p->emsg;
 468       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 469     }
 470 #if DEBUG_PUBLISH
 471   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 472               "Finished with tree encoder\n");
 473 #endif
 474   /* final progress event */
 475   flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
 476   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 477   pi.value.publish.specifics.progress.data = NULL;
 478   pi.value.publish.specifics.progress.offset = flen;
 479   pi.value.publish.specifics.progress.data_len = 0;
 480   pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
 481   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen);
 482
 483   /* continue with main */
 484   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 485   pc->upload_task
 486     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 487                                           &GNUNET_FS_publish_main_,
 488                                           pc);
 489 }
 490
 491
 492 /**
 493  * Function called asking for the current (encoded)
 494  * block to be processed.  After processing the
 495  * client should either call "GNUNET_FS_tree_encode_next"
 496  * or (on error) "GNUNET_FS_tree_encode_finish".
 497  *
 498  * @param cls closure
 499  * @param chk content hash key for the block
 500  * @param offset offset of the block in the file
 501  * @param depth depth of the block in the file, 0 for DBLOCK
 502  * @param type type of the block (IBLOCK or DBLOCK)
 503  * @param block the (encrypted) block
 504  * @param block_size size of block (in bytes)
 505  */
 506 static void
 507 block_proc (void *cls,
 508             const struct ContentHashKey *chk,
 509             uint64_t offset,
 510             unsigned int depth,
 511             enum GNUNET_BLOCK_Type type,
 512             const void *block,
 513             uint16_t block_size)
 514 {
 515   struct GNUNET_FS_PublishContext *pc = cls;
 516   struct GNUNET_FS_FileInformation *p;
 517   struct OnDemandBlock odb;
 518
 519   p = pc->fi_pos;
 520   if (NULL == pc->dsh)
 521     {
 522 #if DEBUG_PUBLISH
 523       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 524                   "Waiting for datastore connection\n");
 525 #endif
 526       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 527       pc->upload_task
 528         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 529                                               &GNUNET_FS_publish_main_,
 530                                               pc);
 531       return;
 532     }
 533
 534   GNUNET_assert (GNUNET_NO == pc->in_network_wait);
 535   pc->in_network_wait = GNUNET_YES;
 536   if ( (! p->is_directory) &&
 537        (GNUNET_YES == p->data.file.do_index) &&
 538        (type == GNUNET_BLOCK_TYPE_FS_DBLOCK) )
 539     {
 540 #if DEBUG_PUBLISH
 541       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 542                   "Indexing block `%s' for offset %llu with index size %u\n",
 543                   GNUNET_h2s (&chk->query),
 544                   (unsigned long long) offset,
 545                   sizeof (struct OnDemandBlock));
 546 #endif
 547       odb.offset = GNUNET_htonll (offset);
 548       odb.file_id = p->data.file.file_id;
 549       GNUNET_assert (pc->qre == NULL);
 550       pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 551                                       (p->is_directory) ? 0 : pc->rid,
 552                                       &chk->query,
 553                                       sizeof (struct OnDemandBlock),
 554                                       &odb,
 555                                       GNUNET_BLOCK_TYPE_FS_ONDEMAND,
 556                                       p->bo.content_priority,
 557                                       p->bo.anonymity_level,
 558                                       p->bo.replication_level,
 559                                       p->bo.expiration_time,
 560                                       -2, 1,
 561                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 562                                       &ds_put_cont,
 563                                       pc);
 564       return;
 565     }
 566 #if DEBUG_PUBLISH
 567   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 568               "Publishing block `%s' for offset %llu with size %u\n",
 569               GNUNET_h2s (&chk->query),
 570               (unsigned long long) offset,
 571               (unsigned int) block_size);
 572 #endif
 573   GNUNET_assert (pc->qre == NULL);
 574   pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 575                                   (p->is_directory) ? 0 : pc->rid,
 576                                   &chk->query,
 577                                   block_size,
 578                                   block,
 579                                   type,
 580                                   p->bo.content_priority,
 581                                   p->bo.anonymity_level,
 582                                   p->bo.replication_level,
 583                                   p->bo.expiration_time,
 584                                   -2, 1,
 585                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 586                                   &ds_put_cont,
 587                                   pc);
 588 }
 589
 590
 591 /**
 592  * Function called with information about our
 593  * progress in computing the tree encoding.
 594  *
 595  * @param cls closure
 596  * @param offset where are we in the file
 597  * @param pt_block plaintext of the currently processed block
 598  * @param pt_size size of pt_block
 599  * @param depth depth of the block in the tree, 0 for DBLOCK
 600  */
 601 static void
 602 progress_proc (void *cls,
 603                uint64_t offset,
 604                const void *pt_block,
 605                size_t pt_size,
 606                unsigned int depth)
 607 {
 608   struct GNUNET_FS_PublishContext *pc = cls;
 609   struct GNUNET_FS_FileInformation *p;
 610   struct GNUNET_FS_ProgressInfo pi;
 611
 612   p = pc->fi_pos;
 613   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 614   pi.value.publish.specifics.progress.data = pt_block;
 615   pi.value.publish.specifics.progress.offset = offset;
 616   pi.value.publish.specifics.progress.data_len = pt_size;
 617   pi.value.publish.specifics.progress.depth = depth;
 618   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset);
 619 }
 620
 621
 622 /**
 623  * We are uploading a file or directory; load (if necessary) the next
 624  * block into memory, encrypt it and send it to the FS service.  Then
 625  * continue with the main task.
 626  *
 627  * @param pc overall upload data
 628  */
 629 static void
 630 publish_content (struct GNUNET_FS_PublishContext *pc)
 631 {
 632   struct GNUNET_FS_FileInformation *p;
 633   char *emsg;
 634   struct GNUNET_FS_DirectoryBuilder *db;
 635   struct GNUNET_FS_FileInformation *dirpos;
 636   void *raw_data;
 637   uint64_t size;
 638
 639   p = pc->fi_pos;
 640   GNUNET_assert (p != NULL);
 641   if (NULL == p->te)
 642     {
 643       if (p->is_directory)
 644         {
 645 #if DEBUG_PUBLISH
 646           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 647                       "Creating directory\n");
 648 #endif
 649           db = GNUNET_FS_directory_builder_create (p->meta);
 650           dirpos = p->data.dir.entries;
 651           while (NULL != dirpos)
 652             {
 653               if (dirpos->is_directory)
 654                 {
 655                   raw_data = dirpos->data.dir.dir_data;
 656                   dirpos->data.dir.dir_data = NULL;
 657                 }
 658               else
 659                 {
 660                   raw_data = NULL;
 661                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 662                        (dirpos->data.file.file_size > 0) )
 663                     {
 664                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 665                       emsg = NULL;
 666                       if (dirpos->data.file.file_size !=
 667                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 668                                                     0,
 669                                                     dirpos->data.file.file_size,
 670                                                     raw_data,
 671                                                     &emsg))
 672                         {
 673                           GNUNET_free_non_null (emsg);
 674                           GNUNET_free (raw_data);
 675                           raw_data = NULL;
 676                         }
 677                     }
 678                 }
 679               GNUNET_FS_directory_builder_add (db,
 680                                                dirpos->chk_uri,
 681                                                dirpos->meta,
 682                                                raw_data);
 683               GNUNET_free_non_null (raw_data);
 684               dirpos = dirpos->next;
 685             }
 686           GNUNET_FS_directory_builder_finish (db,
 687                                               &p->data.dir.dir_size,
 688                                               &p->data.dir.dir_data);
 689           GNUNET_FS_file_information_sync_ (p);
 690         }
 691       size = (p->is_directory)
 692         ? p->data.dir.dir_size
 693         : p->data.file.file_size;
 694 #if DEBUG_PUBLISH
 695       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 696                   "Creating tree encoder\n");
 697 #endif
 698       p->te = GNUNET_FS_tree_encoder_create (pc->h,
 699                                              size,
 700                                              pc,
 701                                              &block_reader,
 702                                              &block_proc,
 703                                              &progress_proc,
 704                                              &encode_cont);
 705
 706     }
 707 #if DEBUG_PUBLISH
 708   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 709               "Processing next block from tree\n");
 710 #endif
 711   GNUNET_FS_tree_encoder_next (p->te);
 712 }
 713
 714
 715 /**
 716  * Process the response (or lack thereof) from
 717  * the "fs" service to our 'start index' request.
 718  *
 719  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 720  * @param msg the response we got
 721  */
 722 static void
 723 process_index_start_response (void *cls,
 724                               const struct GNUNET_MessageHeader *msg)
 725 {
 726   struct GNUNET_FS_PublishContext *pc = cls;
 727   struct GNUNET_FS_FileInformation *p;
 728   const char *emsg;
 729   uint16_t msize;
 730
 731   GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 732   pc->client = NULL;
 733   p = pc->fi_pos;
 734   if (msg == NULL)
 735     {
 736       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 737                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 738                   p->filename,
 739                   _("timeout on index-start request to `fs' service"));
 740       p->data.file.do_index = GNUNET_NO;
 741       GNUNET_FS_file_information_sync_ (p);
 742       publish_content (pc);
 743       return;
 744     }
 745   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 746     {
 747       msize = ntohs (msg->size);
 748       emsg = (const char *) &msg[1];
 749       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 750            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 751         emsg = gettext_noop ("unknown error");
 752       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 753                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 754                   p->filename,
 755                   gettext (emsg));
 756       p->data.file.do_index = GNUNET_NO;
 757       GNUNET_FS_file_information_sync_ (p);
 758       publish_content (pc);
 759       return;
 760     }
 761   p->data.file.index_start_confirmed = GNUNET_YES;
 762   /* success! continue with indexing */
 763   GNUNET_FS_file_information_sync_ (p);
 764   publish_content (pc);
 765 }
 766
 767
 768 /**
 769  * Function called once the hash computation over an
 770  * indexed file has completed.
 771  *
 772  * @param cls closure, our publishing context
 773  * @param res resulting hash, NULL on error
 774  */
 775 static void
 776 hash_for_index_cb (void *cls,
 777                    const GNUNET_HashCode *
 778                    res)
 779 {
 780   struct GNUNET_FS_PublishContext *pc = cls;
 781   struct GNUNET_FS_FileInformation *p;
 782   struct IndexStartMessage *ism;
 783   size_t slen;
 784   struct GNUNET_CLIENT_Connection *client;
 785   uint64_t dev;
 786   uint64_t ino;
 787   char *fn;
 788
 789   pc->fhc = NULL;
 790   p = pc->fi_pos;
 791   if (NULL == res)
 792     {
 793       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 794                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 795                   p->filename,
 796                   _("failed to compute hash"));
 797       p->data.file.do_index = GNUNET_NO;
 798       GNUNET_FS_file_information_sync_ (p);
 799       publish_content (pc);
 800       return;
 801     }
 802   if (GNUNET_YES == p->data.file.index_start_confirmed)
 803     {
 804       publish_content (pc);
 805       return;
 806     }
 807   fn = GNUNET_STRINGS_filename_expand (p->filename);
 808   GNUNET_assert (fn != NULL);
 809   slen = strlen (fn) + 1;
 810   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 811     {
 812       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 813                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 814                   fn,
 815                   _("filename too long"));
 816       GNUNET_free (fn);
 817       p->data.file.do_index = GNUNET_NO;
 818       GNUNET_FS_file_information_sync_ (p);
 819       publish_content (pc);
 820       return;
 821     }
 822 #if DEBUG_PUBLISH
 823   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 824               "Hash of indexed file `%s' is `%s'\n",
 825               p->filename,
 826               GNUNET_h2s (res));
 827 #endif
 828   if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
 829     {
 830       p->data.file.file_id = *res;
 831       p->data.file.have_hash = GNUNET_YES;
 832       p->data.file.index_start_confirmed = GNUNET_YES;
 833       GNUNET_FS_file_information_sync_ (p);
 834       publish_content (pc);
 835       GNUNET_free (fn);
 836       return;
 837     }
 838   client = GNUNET_CLIENT_connect ("fs",
 839                                   pc->h->cfg);
 840   if (NULL == client)
 841     {
 842       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 843                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 844                   p->filename,
 845                   _("could not connect to `fs' service"));
 846       p->data.file.do_index = GNUNET_NO;
 847       publish_content (pc);
 848       GNUNET_free (fn);
 849       return;
 850     }
 851   if (p->data.file.have_hash != GNUNET_YES)
 852     {
 853       p->data.file.file_id = *res;
 854       p->data.file.have_hash = GNUNET_YES;
 855       GNUNET_FS_file_information_sync_ (p);
 856     }
 857   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 858                        slen);
 859   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 860                            slen);
 861   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 862   if (GNUNET_OK ==
 863       GNUNET_DISK_file_get_identifiers (p->filename,
 864                                         &dev,
 865                                         &ino))
 866     {
 867       ism->device = GNUNET_htonll (dev);
 868       ism->inode = GNUNET_htonll(ino);
 869     }
 870 #if DEBUG_PUBLISH
 871   else
 872     {
 873       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 874                   _("Failed to get file identifiers for `%s'\n"),
 875                   p->filename);
 876     }
 877 #endif
 878   ism->file_id = *res;
 879   memcpy (&ism[1],
 880           fn,
 881           slen);
 882   GNUNET_free (fn);
 883   pc->client = client;
 884   GNUNET_break (GNUNET_YES ==
 885                 GNUNET_CLIENT_transmit_and_get_response (client,
 886                                                          &ism->header,
 887                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 888                                                          GNUNET_YES,
 889                                                          &process_index_start_response,
 890                                                          pc));
 891   GNUNET_free (ism);
 892 }
 893
 894
 895 /**
 896  * Main function that performs the upload.
 897  *
 898  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 899  * @param tc task context
 900  */
 901 void
 902 GNUNET_FS_publish_main_ (void *cls,
 903                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 904 {
 905   struct GNUNET_FS_PublishContext *pc = cls;
 906   struct GNUNET_FS_ProgressInfo pi;
 907   struct GNUNET_FS_FileInformation *p;
 908   struct GNUNET_FS_Uri *loc;
 909   char *fn;
 910
 911   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 912   p = pc->fi_pos;
 913   if (NULL == p)
 914     {
 915 #if DEBUG_PUBLISH
 916       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 917                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 918 #endif
 919       /* upload of entire hierarchy complete,
 920          publish namespace entries */
 921       GNUNET_FS_publish_sync_ (pc);
 922       publish_sblock (pc);
 923       return;
 924     }
 925   /* find starting position */
 926   while ( (p->is_directory) &&
 927           (NULL != p->data.dir.entries) &&
 928           (NULL == p->emsg) &&
 929           (NULL == p->data.dir.entries->chk_uri) )
 930     {
 931       p = p->data.dir.entries;
 932       pc->fi_pos = p;
 933       GNUNET_FS_publish_sync_ (pc);
 934     }
 935   /* abort on error */
 936   if (NULL != p->emsg)
 937     {
 938 #if DEBUG_PUBLISH
 939       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 940                   "Error uploading: %s\n",
 941                   p->emsg);
 942 #endif
 943       /* error with current file, abort all
 944          related files as well! */
 945       while (NULL != p->dir)
 946         {
 947           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 948                                                        EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
 949           p = p->dir;
 950           if (fn != NULL)
 951             {
 952               GNUNET_asprintf (&p->emsg,
 953                                _("Recursive upload failed at `%s': %s"),
 954                                fn,
 955                                p->emsg);
 956               GNUNET_free (fn);
 957             }
 958           else
 959             {
 960               GNUNET_asprintf (&p->emsg,
 961                                _("Recursive upload failed: %s"),
 962                                p->emsg);
 963             }
 964           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 965           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 966           pi.value.publish.specifics.error.message = p->emsg;
 967           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 968         }
 969       pc->all_done = GNUNET_YES;
 970       GNUNET_FS_publish_sync_ (pc);
 971       return;
 972     }
 973   /* handle completion */
 974   if (NULL != p->chk_uri)
 975     {
 976 #if DEBUG_PUBLISH
 977       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 978                   "File upload complete, now publishing KSK blocks.\n");
 979 #endif
 980       if (0 == p->bo.anonymity_level)
 981         {
 982           /* zero anonymity, box CHK URI in LOC URI */
 983           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
 984                                           pc->h->cfg,
 985                                           p->bo.expiration_time);
 986           GNUNET_FS_uri_destroy (p->chk_uri);
 987           p->chk_uri = loc;
 988         }
 989       GNUNET_FS_publish_sync_ (pc);
 990       /* upload of "p" complete, publish KBlocks! */
 991       if (p->keywords != NULL)
 992         {
 993           GNUNET_FS_publish_ksk (pc->h,
 994                                  p->keywords,
 995                                  p->meta,
 996                                  p->chk_uri,
 997                                  &p->bo,
 998                                  pc->options,
 999                                  &publish_kblocks_cont,
1000                                  pc);
1001         }
1002       else
1003         {
1004           publish_kblocks_cont (pc,
1005                                 p->chk_uri,
1006                                 NULL);
1007         }
1008       return;
1009     }
1010   if ( (!p->is_directory) &&
1011        (p->data.file.do_index) )
1012     {
1013       if (NULL == p->filename)
1014         {
1015           p->data.file.do_index = GNUNET_NO;
1016           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1017                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1018                       "<no-name>",
1019                       _("needs to be an actual file"));
1020           GNUNET_FS_file_information_sync_ (p);
1021           publish_content (pc);
1022           return;
1023         }
1024       if (p->data.file.have_hash)
1025         {
1026           hash_for_index_cb (pc,
1027                              &p->data.file.file_id);
1028         }
1029       else
1030         {
1031           p->start_time = GNUNET_TIME_absolute_get ();
1032           pc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
1033                                              p->filename,
1034                                              HASHING_BLOCKSIZE,
1035                                              &hash_for_index_cb,
1036                                              pc);
1037         }
1038       return;
1039     }
1040   publish_content (pc);
1041 }
1042
1043
1044 /**
1045  * Signal the FS's progress function that we are starting
1046  * an upload.
1047  *
1048  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1049  * @param fi the entry in the publish-structure
1050  * @param length length of the file or directory
1051  * @param meta metadata for the file or directory (can be modified)
1052  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1053  * @param bo block options
1054  * @param do_index should we index?
1055  * @param client_info pointer to client context set upon creation (can be modified)
1056  * @return GNUNET_OK to continue (always)
1057  */
1058 static int
1059 fip_signal_start(void *cls,
1060                  struct GNUNET_FS_FileInformation *fi,
1061                  uint64_t length,
1062                  struct GNUNET_CONTAINER_MetaData *meta,
1063                  struct GNUNET_FS_Uri **uri,
1064                  struct GNUNET_FS_BlockOptions *bo,
1065                  int *do_index,
1066                  void **client_info)
1067 {
1068   struct GNUNET_FS_PublishContext *pc = cls;
1069   struct GNUNET_FS_ProgressInfo pi;
1070   unsigned int kc;
1071   uint64_t left;
1072
1073 #if DEBUG_PUBLISH
1074   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1075               "Starting publish operation\n");
1076 #endif
1077   if (*do_index)
1078     {
1079       /* space for on-demand blocks */
1080       pc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1081     }
1082   else
1083     {
1084       /* space for DBlocks */
1085       pc->reserve_space += length;
1086     }
1087   /* entries for IBlocks and DBlocks, space for IBlocks */
1088   left = length;
1089   while (1)
1090     {
1091       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1092       pc->reserve_entries += left;
1093       if (left <= 1)
1094         break;
1095       left = left * sizeof (struct ContentHashKey);
1096       pc->reserve_space += left;
1097     }
1098   pc->reserve_entries++;
1099   /* entries and space for keywords */
1100   if (NULL != *uri)
1101     {
1102       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1103       pc->reserve_entries += kc;
1104       pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1105     }
1106   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1107   *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
1108   GNUNET_FS_file_information_sync_ (fi);
1109   return GNUNET_OK;
1110 }
1111
1112
1113 /**
1114  * Signal the FS's progress function that we are suspending
1115  * an upload.
1116  *
1117  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1118  * @param fi the entry in the publish-structure
1119  * @param length length of the file or directory
1120  * @param meta metadata for the file or directory (can be modified)
1121  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1122  * @param bo block options
1123  * @param do_index should we index?
1124  * @param client_info pointer to client context set upon creation (can be modified)
1125  * @return GNUNET_OK to continue (always)
1126  */
1127 static int
1128 fip_signal_suspend(void *cls,
1129                    struct GNUNET_FS_FileInformation *fi,
1130                    uint64_t length,
1131                    struct GNUNET_CONTAINER_MetaData *meta,
1132                    struct GNUNET_FS_Uri **uri,
1133                    struct GNUNET_FS_BlockOptions *bo,
1134                    int *do_index,
1135                    void **client_info)
1136 {
1137   struct GNUNET_FS_PublishContext*pc = cls;
1138   struct GNUNET_FS_ProgressInfo pi;
1139   uint64_t off;
1140
1141 #if DEBUG_PUBLISH
1142   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1143               "Suspending publish operation\n");
1144 #endif
1145   GNUNET_free_non_null (fi->serialization);
1146   fi->serialization = NULL;
1147   off = (fi->chk_uri == NULL) ? 0 : length;
1148   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1149   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1150   *client_info = NULL;
1151   if (NULL != pc->qre)
1152     {
1153       GNUNET_DATASTORE_cancel (pc->qre);
1154       pc->qre = NULL;
1155     }
1156   if (NULL != pc->dsh)
1157     {
1158       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1159       pc->dsh = NULL;
1160     }
1161   pc->rid = 0;
1162   return GNUNET_OK;
1163 }
1164
1165
1166 /**
1167  * Create SUSPEND event for the given publish operation
1168  * and then clean up our state (without stop signal).
1169  *
1170  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1171  */
1172 void
1173 GNUNET_FS_publish_signal_suspend_ (void *cls)
1174 {
1175   struct GNUNET_FS_PublishContext *pc = cls;
1176
1177   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1178     {
1179       GNUNET_SCHEDULER_cancel (pc->upload_task);
1180       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1181     }
1182   GNUNET_FS_file_information_inspect (pc->fi,
1183                                       &fip_signal_suspend,
1184                                       pc);
1185   GNUNET_FS_end_top (pc->h, pc->top);
1186   pc->top = NULL;
1187   publish_cleanup (pc, NULL);
1188 }
1189
1190
1191 /**
1192  * We have gotten a reply for our space reservation request.
1193  * Either fail (insufficient space) or start publishing for good.
1194  *
1195  * @param cls the 'struct GNUNET_FS_PublishContext*'
1196  * @param success positive reservation ID on success
1197  * @param msg error message on error, otherwise NULL
1198  */
1199 static void
1200 finish_reserve (void *cls,
1201                 int success,
1202                 const char *msg)
1203 {
1204   struct GNUNET_FS_PublishContext *pc = cls;
1205
1206   pc->qre = NULL;
1207 #if DEBUG_PUBLISH
1208   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1209               "Reservation complete (%d)!\n",
1210               success);
1211 #endif
1212   if ( (msg != NULL) ||
1213        (success <= 0) )
1214     {
1215       GNUNET_asprintf (&pc->fi->emsg,
1216                        _("Insufficient space for publishing: %s"),
1217                        msg);
1218       signal_publish_error (pc->fi,
1219                             pc,
1220                             pc->fi->emsg);
1221       return;
1222     }
1223   pc->rid = success;
1224   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
1225   pc->upload_task
1226     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1227                                           &GNUNET_FS_publish_main_,
1228                                           pc);
1229 }
1230
1231
1232 /**
1233  * Publish a file or directory.
1234  *
1235  * @param h handle to the file sharing subsystem
1236  * @param fi information about the file or directory structure to publish
1237  * @param namespace namespace to publish the file in, NULL for no namespace
1238  * @param nid identifier to use for the publishd content in the namespace
1239  *        (can be NULL, must be NULL if namespace is NULL)
1240  * @param nuid update-identifier that will be used for future updates
1241  *        (can be NULL, must be NULL if namespace or nid is NULL)
1242  * @param options options for the publication
1243  * @return context that can be used to control the publish operation
1244  */
1245 struct GNUNET_FS_PublishContext *
1246 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1247                          struct GNUNET_FS_FileInformation *fi,
1248                          struct GNUNET_FS_Namespace *namespace,
1249                          const char *nid,
1250                          const char *nuid,
1251                          enum GNUNET_FS_PublishOptions options)
1252 {
1253   struct GNUNET_FS_PublishContext *ret;
1254   struct GNUNET_DATASTORE_Handle *dsh;
1255
1256   GNUNET_assert (NULL != h);
1257   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1258     {
1259       dsh = GNUNET_DATASTORE_connect (h->cfg);
1260       if (NULL == dsh)
1261         return NULL;
1262     }
1263   else
1264     {
1265       dsh = NULL;
1266     }
1267   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1268   ret->dsh = dsh;
1269   ret->h = h;
1270   ret->fi = fi;
1271   ret->namespace = namespace;
1272   ret->options = options;
1273   if (namespace != NULL)
1274     {
1275       namespace->rc++;
1276       GNUNET_assert (NULL != nid);
1277       ret->nid = GNUNET_strdup (nid);
1278       if (NULL != nuid)
1279         ret->nuid = GNUNET_strdup (nuid);
1280     }
1281   /* signal start */
1282   GNUNET_FS_file_information_inspect (ret->fi,
1283                                       &fip_signal_start,
1284                                       ret);
1285   ret->fi_pos = ret->fi;
1286   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1287   GNUNET_FS_publish_sync_ (ret);
1288   if (NULL != ret->dsh)
1289     {
1290       GNUNET_assert (NULL == ret->qre);
1291       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1292                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1293                   (unsigned int) ret->reserve_entries,
1294                   (unsigned long long) ret->reserve_space);
1295       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1296                                            ret->reserve_space,
1297                                            ret->reserve_entries,
1298                                            UINT_MAX,
1299                                            UINT_MAX,
1300                                            GNUNET_TIME_UNIT_FOREVER_REL,
1301                                            &finish_reserve,
1302                                            ret);
1303     }
1304   else
1305     {
1306       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == ret->upload_task);
1307       ret->upload_task
1308         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1309                                               &GNUNET_FS_publish_main_,
1310                                               ret);
1311     }
1312   return ret;
1313 }
1314
1315
1316 /**
1317  * Signal the FS's progress function that we are stopping
1318  * an upload.
1319  *
1320  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1321  * @param fi the entry in the publish-structure
1322  * @param length length of the file or directory
1323  * @param meta metadata for the file or directory (can be modified)
1324  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1325  * @param bo block options (can be modified)
1326  * @param do_index should we index?
1327  * @param client_info pointer to client context set upon creation (can be modified)
1328  * @return GNUNET_OK to continue (always)
1329  */
1330 static int
1331 fip_signal_stop(void *cls,
1332                 struct GNUNET_FS_FileInformation *fi,
1333                 uint64_t length,
1334                 struct GNUNET_CONTAINER_MetaData *meta,
1335                 struct GNUNET_FS_Uri **uri,
1336                 struct GNUNET_FS_BlockOptions *bo,
1337                 int *do_index,
1338                 void **client_info)
1339 {
1340   struct GNUNET_FS_PublishContext*pc = cls;
1341   struct GNUNET_FS_ProgressInfo pi;
1342   uint64_t off;
1343
1344   if (fi->serialization != NULL)
1345     {
1346       GNUNET_FS_remove_sync_file_ (pc->h,
1347                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1348                                    fi->serialization);
1349       GNUNET_free (fi->serialization);
1350       fi->serialization = NULL;
1351     }
1352   off = (fi->chk_uri == NULL) ? 0 : length;
1353   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1354   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1355   *client_info = NULL;
1356   return GNUNET_OK;
1357 }
1358
1359
1360 /**
1361  * Stop an upload.  Will abort incomplete uploads (but
1362  * not remove blocks that have already been publishd) or
1363  * simply clean up the state for completed uploads.
1364  * Must NOT be called from within the event callback!
1365  *
1366  * @param pc context for the upload to stop
1367  */
1368 void
1369 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1370 {
1371 #if DEBUG_PUBLISH
1372   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1373               "Publish stop called\n");
1374 #endif
1375   GNUNET_FS_end_top (pc->h, pc->top);
1376   if (NULL != pc->qre)
1377     {
1378       GNUNET_DATASTORE_cancel (pc->qre);
1379       pc->qre = NULL;
1380     }
1381   if (NULL != pc->dsh)
1382     {
1383       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1384       pc->dsh = NULL;
1385     }
1386   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1387     {
1388       GNUNET_SCHEDULER_cancel (pc->upload_task);
1389       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1390     }
1391   if (pc->serialization != NULL)
1392     {
1393       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1394       GNUNET_free (pc->serialization);
1395       pc->serialization = NULL;
1396     }
1397   GNUNET_FS_file_information_inspect (pc->fi,
1398                                       &fip_signal_stop,
1399                                       pc);
1400   if (GNUNET_YES == pc->in_network_wait)
1401     {
1402       pc->in_network_wait = GNUNET_SYSERR;
1403       return;
1404     }
1405   publish_cleanup (pc, NULL);
1406 }
1407
1408
1409 /**
1410  * Context for the KSK publication.
1411  */
1412 struct PublishKskContext
1413 {
1414
1415   /**
1416    * Keywords to use.
1417    */
1418   struct GNUNET_FS_Uri *ksk_uri;
1419
1420   /**
1421    * Global FS context.
1422    */
1423   struct GNUNET_FS_Handle *h;
1424
1425   /**
1426    * The master block that we are sending
1427    * (in plaintext), has "mdsize+slen" more
1428    * bytes than the struct would suggest.
1429    */
1430   struct KBlock *kb;
1431
1432   /**
1433    * Buffer of the same size as "kb" for
1434    * the encrypted version.
1435    */
1436   struct KBlock *cpy;
1437
1438   /**
1439    * Handle to the datastore, NULL if we are just
1440    * simulating.
1441    */
1442   struct GNUNET_DATASTORE_Handle *dsh;
1443
1444   /**
1445    * Handle to datastore PUT request.
1446    */
1447   struct GNUNET_DATASTORE_QueueEntry *qre;
1448
1449   /**
1450    * Function to call once we're done.
1451    */
1452   GNUNET_FS_PublishContinuation cont;
1453
1454   /**
1455    * Closure for cont.
1456    */
1457   void *cont_cls;
1458
1459   /**
1460    * When should the KBlocks expire?
1461    */
1462   struct GNUNET_FS_BlockOptions bo;
1463
1464   /**
1465    * Size of the serialized metadata.
1466    */
1467   ssize_t mdsize;
1468
1469   /**
1470    * Size of the (CHK) URI as a string.
1471    */
1472   size_t slen;
1473
1474   /**
1475    * Keyword that we are currently processing.
1476    */
1477   unsigned int i;
1478
1479 };
1480
1481
1482 /**
1483  * Continuation of "GNUNET_FS_publish_ksk" that performs
1484  * the actual publishing operation (iterating over all
1485  * of the keywords).
1486  *
1487  * @param cls closure of type "struct PublishKskContext*"
1488  * @param tc unused
1489  */
1490 static void
1491 publish_ksk_cont (void *cls,
1492                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1493
1494
1495 /**
1496  * Function called by the datastore API with
1497  * the result from the PUT request.
1498  *
1499  * @param cls closure of type "struct PublishKskContext*"
1500  * @param success GNUNET_OK on success
1501  * @param msg error message (or NULL)
1502  */
1503 static void
1504 kb_put_cont (void *cls,
1505              int success,
1506              const char *msg)
1507 {
1508   struct PublishKskContext *pkc = cls;
1509
1510   pkc->qre = NULL;
1511   if (GNUNET_OK != success)
1512     {
1513 #if DEBUG_PUBLISH
1514       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1515                   "KB PUT operation complete\n");
1516 #endif
1517       if (NULL != pkc->dsh)
1518         {
1519           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1520           pkc->dsh = NULL;
1521         }
1522       GNUNET_free (pkc->cpy);
1523       GNUNET_free (pkc->kb);
1524       pkc->cont (pkc->cont_cls,
1525                  NULL,
1526                  msg);
1527       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1528       GNUNET_free (pkc);
1529       return;
1530     }
1531   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1532                                      pkc,
1533                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1534 }
1535
1536
1537 /**
1538  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1539  * publishing operation (iterating over all of the keywords).
1540  *
1541  * @param cls closure of type "struct PublishKskContext*"
1542  * @param tc unused
1543  */
1544 static void
1545 publish_ksk_cont (void *cls,
1546                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1547 {
1548   struct PublishKskContext *pkc = cls;
1549   const char *keyword;
1550   GNUNET_HashCode key;
1551   GNUNET_HashCode query;
1552   struct GNUNET_CRYPTO_AesSessionKey skey;
1553   struct GNUNET_CRYPTO_AesInitializationVector iv;
1554   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1555
1556
1557   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1558        (NULL == pkc->dsh) )
1559     {
1560 #if DEBUG_PUBLISH
1561       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1562                   "KSK PUT operation complete\n");
1563 #endif
1564       if (NULL != pkc->dsh)
1565         {
1566           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1567           pkc->dsh = NULL;
1568         }
1569       GNUNET_free (pkc->cpy);
1570       GNUNET_free (pkc->kb);
1571       pkc->cont (pkc->cont_cls,
1572                  pkc->ksk_uri,
1573                  NULL);
1574       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1575       GNUNET_free (pkc);
1576       return;
1577     }
1578   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1579 #if DEBUG_PUBLISH
1580   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1581               "Publishing under keyword `%s'\n",
1582               keyword);
1583 #endif
1584   /* first character of keyword indicates if it is
1585      mandatory or not -- ignore for hashing */
1586   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1587   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1588   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1589                              pkc->slen + pkc->mdsize,
1590                              &skey,
1591                              &iv,
1592                              &pkc->cpy[1]);
1593   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1594   GNUNET_assert (NULL != pk);
1595   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1596   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1597                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1598                       &query);
1599   GNUNET_assert (GNUNET_OK ==
1600                  GNUNET_CRYPTO_rsa_sign (pk,
1601                                          &pkc->cpy->purpose,
1602                                          &pkc->cpy->signature));
1603   GNUNET_CRYPTO_rsa_key_free (pk);
1604   pkc->qre = GNUNET_DATASTORE_put (pkc->dsh,
1605                                    0,
1606                                    &query,
1607                                    pkc->mdsize +
1608                                    sizeof (struct KBlock) +
1609                                    pkc->slen,
1610                                    pkc->cpy,
1611                                    GNUNET_BLOCK_TYPE_FS_KBLOCK,
1612                                    pkc->bo.content_priority,
1613                                    pkc->bo.anonymity_level,
1614                                    pkc->bo.replication_level,
1615                                    pkc->bo.expiration_time,
1616                                    -2, 1,
1617                                    GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1618                                    &kb_put_cont,
1619                                    pkc);
1620 }
1621
1622
1623 /**
1624  * Publish a CHK under various keywords on GNUnet.
1625  *
1626  * @param h handle to the file sharing subsystem
1627  * @param ksk_uri keywords to use
1628  * @param meta metadata to use
1629  * @param uri URI to refer to in the KBlock
1630  * @param bo per-block options
1631  * @param options publication options
1632  * @param cont continuation
1633  * @param cont_cls closure for cont
1634  */
1635 void
1636 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1637                        const struct GNUNET_FS_Uri *ksk_uri,
1638                        const struct GNUNET_CONTAINER_MetaData *meta,
1639                        const struct GNUNET_FS_Uri *uri,
1640                        const struct GNUNET_FS_BlockOptions *bo,
1641                        enum GNUNET_FS_PublishOptions options,
1642                        GNUNET_FS_PublishContinuation cont,
1643                        void *cont_cls)
1644 {
1645   struct PublishKskContext *pkc;
1646   char *uris;
1647   size_t size;
1648   char *kbe;
1649   char *sptr;
1650
1651   GNUNET_assert (NULL != uri);
1652   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1653   pkc->h = h;
1654   pkc->bo = *bo;
1655   pkc->cont = cont;
1656   pkc->cont_cls = cont_cls;
1657   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1658     {
1659       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
1660       if (pkc->dsh == NULL)
1661         {
1662           cont (cont_cls, NULL, _("Could not connect to datastore."));
1663           GNUNET_free (pkc);
1664           return;
1665         }
1666     }
1667   if (meta == NULL)
1668     pkc->mdsize = 0;
1669   else
1670     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1671   GNUNET_assert (pkc->mdsize >= 0);
1672   uris = GNUNET_FS_uri_to_string (uri);
1673   pkc->slen = strlen (uris) + 1;
1674   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1675   if (size > MAX_KBLOCK_SIZE)
1676     {
1677       size = MAX_KBLOCK_SIZE;
1678       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1679     }
1680   pkc->kb = GNUNET_malloc (size);
1681   kbe = (char *) &pkc->kb[1];
1682   memcpy (kbe, uris, pkc->slen);
1683   GNUNET_free (uris);
1684   sptr = &kbe[pkc->slen];
1685   if (meta != NULL)
1686     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1687                                                         &sptr,
1688                                                         pkc->mdsize,
1689                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1690   if (pkc->mdsize == -1)
1691     {
1692       GNUNET_break (0);
1693       GNUNET_free (pkc->kb);
1694       if (pkc->dsh != NULL)
1695         {
1696           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1697           pkc->dsh = NULL;
1698         }
1699       cont (cont_cls, NULL, _("Internal error."));
1700       GNUNET_free (pkc);
1701       return;
1702     }
1703   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1704
1705   pkc->cpy = GNUNET_malloc (size);
1706   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1707                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1708                                   pkc->mdsize +
1709                                   pkc->slen);
1710   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1711   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1712   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1713                                      pkc,
1714                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1715 }
1716
1717
1718 /* end of fs_publish.c */