src/fs/fs_publish.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file fs/fs_publish.c
  23  * @brief publish a file or directory in GNUnet
  24  * @see https://gnunet.org/encoding
  25  * @author Krista Bennett
  26  * @author Christian Grothoff
  27  */
  28
  29 #include "platform.h"
  30 #include "gnunet_constants.h"
  31 #include "gnunet_signatures.h"
  32 #include "gnunet_util_lib.h"
  33 #include "gnunet_fs_service.h"
  34 #include "fs.h"
  35 #include "fs_tree.h"
  36
  37 #define DEBUG_PUBLISH GNUNET_NO
  38
  39
  40 /**
  41  * Context for "ds_put_cont".
  42  */
  43 struct PutContCtx
  44 {
  45   /**
  46    * Current publishing context.
  47    */
  48   struct GNUNET_FS_PublishContext *pc;
  49
  50   /**
  51    * Specific file with the block.
  52    */
  53   struct GNUNET_FS_FileInformation *p;
  54
  55   /**
  56    * Function to run next, if any (can be NULL).
  57    */
  58   GNUNET_SCHEDULER_Task cont;
  59
  60   /**
  61    * Closure for cont.
  62    */
  63   void *cont_cls;
  64 };
  65
  66
  67 /**
  68  * Fill in all of the generic fields for
  69  * a publish event and call the callback.
  70  *
  71  * @param pi structure to fill in
  72  * @param pc overall publishing context
  73  * @param p file information for the file being published
  74  * @param offset where in the file are we so far
  75  * @return value returned from callback
  76  */
  77 void *
  78 GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
  79                                 struct GNUNET_FS_PublishContext *pc,
  80                                 const struct GNUNET_FS_FileInformation *p,
  81                                 uint64_t offset)
  82 {
  83   pi->value.publish.pc = pc;
  84   pi->value.publish.fi = p;
  85   pi->value.publish.cctx
  86     = p->client_info;
  87   pi->value.publish.pctx
  88     = (NULL == p->dir) ? NULL : p->dir->client_info;
  89   pi->value.publish.filename = p->filename;
  90   pi->value.publish.size
  91     = (p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
  92   pi->value.publish.eta
  93     = GNUNET_TIME_calculate_eta (p->start_time,
  94                                  offset,
  95                                  pi->value.publish.size);
  96   pi->value.publish.completed = offset;
  97   pi->value.publish.duration = GNUNET_TIME_absolute_get_duration (p->start_time);
  98   pi->value.publish.anonymity = p->bo.anonymity_level;
  99   return pc->h->upcb (pc->h->upcb_cls,
 100                       pi);
 101 }
 102
 103
 104 /**
 105  * Cleanup the publish context, we're done with it.
 106  *
 107  * @param cls struct to clean up after
 108  * @param tc scheduler context
 109  */
 110 static void
 111 publish_cleanup (void *cls,
 112                  const struct GNUNET_SCHEDULER_TaskContext *tc)
 113 {
 114   struct GNUNET_FS_PublishContext *pc = cls;
 115
 116 #if DEBUG_PUBLISH
 117   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 118               "Cleaning up publish context (done!)\n");
 119 #endif
 120   if (pc->fhc != NULL)
 121     {
 122       GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
 123       pc->fhc = NULL;
 124     }
 125   GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
 126   if (pc->namespace != NULL)
 127     {
 128       GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO);
 129       pc->namespace = NULL;
 130     }
 131   GNUNET_free_non_null (pc->nid);
 132   GNUNET_free_non_null (pc->nuid);
 133   GNUNET_free_non_null (pc->serialization);
 134   if (pc->dsh != NULL)
 135     {
 136       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
 137       pc->dsh = NULL;
 138     }
 139   if (pc->client != NULL)
 140     {
 141       GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 142       pc->client = NULL;
 143     }
 144   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 145   GNUNET_free (pc);
 146 }
 147
 148
 149 /**
 150  * Function called by the datastore API with
 151  * the result from the PUT request.
 152  *
 153  * @param cls our closure
 154  * @param success GNUNET_OK on success
 155  * @param msg error message (or NULL)
 156  */
 157 static void
 158 ds_put_cont (void *cls,
 159              int success,
 160              const char *msg)
 161 {
 162   struct PutContCtx *pcc = cls;
 163   struct GNUNET_FS_ProgressInfo pi;
 164
 165   pcc->pc->qre = NULL;
 166   if (GNUNET_SYSERR == pcc->pc->in_network_wait)
 167     {
 168       /* we were aborted in the meantime, finish shutdown! */
 169       GNUNET_SCHEDULER_add_continuation (&publish_cleanup,
 170                                          pcc->pc,
 171                                          GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 172       GNUNET_free (pcc);
 173       return;
 174     }
 175   GNUNET_assert (GNUNET_YES == pcc->pc->in_network_wait);
 176   pcc->pc->in_network_wait = GNUNET_NO;
 177   if (GNUNET_SYSERR == success)
 178     {
 179       GNUNET_asprintf (&pcc->p->emsg,
 180                        _("Publishing failed: %s"),
 181                        msg);
 182       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 183       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 184       pi.value.publish.specifics.error.message = pcc->p->emsg;
 185       pcc->p->client_info = GNUNET_FS_publish_make_status_ (&pi, pcc->pc, pcc->p, 0);
 186       if ( (pcc->p->is_directory == GNUNET_NO) &&
 187            (pcc->p->filename != NULL) &&
 188            (pcc->p->data.file.do_index == GNUNET_YES) )
 189         {
 190           /* run unindex to clean up */
 191           GNUNET_FS_unindex_start (pcc->pc->h,
 192                                    pcc->p->filename,
 193                                    NULL);
 194         }
 195     }
 196   if (NULL != pcc->cont)
 197     {
 198       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pcc->pc->upload_task);
 199       pcc->pc->upload_task
 200         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 201                                               pcc->cont,
 202                                               pcc->cont_cls);
 203     }
 204   GNUNET_free (pcc);
 205 }
 206
 207
 208 /**
 209  * Generate the callback that signals clients
 210  * that a file (or directory) has been completely
 211  * published.
 212  *
 213  * @param p the completed upload
 214  * @param pc context of the publication
 215  */
 216 static void
 217 signal_publish_completion (struct GNUNET_FS_FileInformation *p,
 218                            struct GNUNET_FS_PublishContext *pc)
 219 {
 220   struct GNUNET_FS_ProgressInfo pi;
 221
 222   pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
 223   pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
 224   pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
 225   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p,
 226                                                    GNUNET_ntohll (p->chk_uri->data.chk.file_length));
 227 }
 228
 229
 230 /**
 231  * Generate the callback that signals clients
 232  * that a file (or directory) has encountered
 233  * a problem during publication.
 234  *
 235  * @param p the upload that had trouble
 236  * @param pc context of the publication
 237  * @param emsg error message
 238  */
 239 static void
 240 signal_publish_error (struct GNUNET_FS_FileInformation *p,
 241                       struct GNUNET_FS_PublishContext *pc,
 242                       const char *emsg)
 243 {
 244   struct GNUNET_FS_ProgressInfo pi;
 245
 246   p->emsg = GNUNET_strdup (emsg);
 247   pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 248   pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 249   pi.value.publish.specifics.error.message =emsg;
 250   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 251   if ( (p->is_directory == GNUNET_NO) &&
 252        (p->filename != NULL) &&
 253        (p->data.file.do_index == GNUNET_YES) )
 254     {
 255       /* run unindex to clean up */
 256       GNUNET_FS_unindex_start (pc->h,
 257                                p->filename,
 258                                NULL);
 259     }
 260
 261 }
 262
 263
 264 /**
 265  * Datastore returns from reservation cancel request.
 266  *
 267  * @param cls the 'struct GNUNET_FS_PublishContext'
 268  * @param success success code (not used)
 269  * @param msg error message (typically NULL, not used)
 270  */
 271 static void
 272 finish_release_reserve (void *cls,
 273                         int success,
 274                         const char *msg)
 275 {
 276   struct GNUNET_FS_PublishContext *pc = cls;
 277
 278   pc->qre = NULL;
 279 #if DEBUG_PUBLISH
 280   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 281               "Releasing reserve done!\n");
 282 #endif
 283   signal_publish_completion (pc->fi, pc);
 284   pc->all_done = GNUNET_YES;
 285   GNUNET_FS_publish_sync_ (pc);
 286 }
 287
 288
 289 /**
 290  * We've finished publishing the SBlock as part of a larger upload.
 291  * Check the result and complete the larger upload.
 292  *
 293  * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload
 294  * @param uri URI of the published SBlock
 295  * @param emsg NULL on success, otherwise error message
 296  */
 297 static void
 298 publish_sblocks_cont (void *cls,
 299                       const struct GNUNET_FS_Uri *uri,
 300                       const char *emsg)
 301 {
 302   struct GNUNET_FS_PublishContext *pc = cls;
 303   if (NULL != emsg)
 304     {
 305       signal_publish_error (pc->fi,
 306                             pc,
 307                             emsg);
 308       GNUNET_FS_publish_sync_ (pc);
 309       return;
 310     }
 311   GNUNET_assert (pc->qre == NULL);
 312   if ( (pc->dsh != NULL) &&
 313        (pc->rid != 0) )
 314     {
 315       pc->qre = GNUNET_DATASTORE_release_reserve (pc->dsh,
 316                                                   pc->rid,
 317                                                   UINT_MAX,
 318                                                   UINT_MAX,
 319                                                   GNUNET_TIME_UNIT_FOREVER_REL,
 320                                                   &finish_release_reserve,
 321                                                   pc);
 322     }
 323   else
 324     {
 325       finish_release_reserve (pc, GNUNET_OK, NULL);
 326     }
 327 }
 328
 329
 330 /**
 331  * We are almost done publishing the structure,
 332  * add SBlocks (if needed).
 333  *
 334  * @param pc overall upload data
 335  */
 336 static void
 337 publish_sblock (struct GNUNET_FS_PublishContext *pc)
 338 {
 339   if (NULL != pc->namespace)
 340     GNUNET_FS_publish_sks (pc->h,
 341                            pc->namespace,
 342                            pc->nid,
 343                            pc->nuid,
 344                            pc->fi->meta,
 345                            pc->fi->chk_uri,
 346                            &pc->fi->bo,
 347                            pc->options,
 348                            &publish_sblocks_cont,
 349                            pc);
 350   else
 351     publish_sblocks_cont (pc, NULL, NULL);
 352 }
 353
 354
 355 /**
 356  * We've finished publishing a KBlock as part of a larger upload.
 357  * Check the result and continue the larger upload.
 358  *
 359  * @param cls the "struct GNUNET_FS_PublishContext*"
 360  *        of the larger upload
 361  * @param uri URI of the published blocks
 362  * @param emsg NULL on success, otherwise error message
 363  */
 364 static void
 365 publish_kblocks_cont (void *cls,
 366                       const struct GNUNET_FS_Uri *uri,
 367                       const char *emsg)
 368 {
 369   struct GNUNET_FS_PublishContext *pc = cls;
 370   struct GNUNET_FS_FileInformation *p = pc->fi_pos;
 371
 372   if (NULL != emsg)
 373     {
 374 #if DEBUG_PUBLISH
 375       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 376                   "Error uploading KSK blocks: %s\n",
 377                   emsg);
 378 #endif
 379       signal_publish_error (p, pc, emsg);
 380       GNUNET_FS_file_information_sync_ (p);
 381       GNUNET_FS_publish_sync_ (pc);
 382       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 383       pc->upload_task
 384         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 385                                               &GNUNET_FS_publish_main_,
 386                                               pc);
 387       return;
 388     }
 389 #if DEBUG_PUBLISH
 390   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 391               "KSK blocks published, moving on to next file\n");
 392 #endif
 393   if (NULL != p->dir)
 394     signal_publish_completion (p, pc);
 395   /* move on to next file */
 396   if (NULL != p->next)
 397     pc->fi_pos = p->next;
 398   else
 399     pc->fi_pos = p->dir;
 400   GNUNET_FS_publish_sync_ (pc);
 401   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 402   pc->upload_task
 403     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 404                                           &GNUNET_FS_publish_main_,
 405                                           pc);
 406 }
 407
 408
 409 /**
 410  * Function called by the tree encoder to obtain
 411  * a block of plaintext data (for the lowest level
 412  * of the tree).
 413  *
 414  * @param cls our publishing context
 415  * @param offset identifies which block to get
 416  * @param max (maximum) number of bytes to get; returning
 417  *        fewer will also cause errors
 418  * @param buf where to copy the plaintext buffer
 419  * @param emsg location to store an error message (on error)
 420  * @return number of bytes copied to buf, 0 on error
 421  */
 422 static size_t
 423 block_reader (void *cls,
 424               uint64_t offset,
 425               size_t max,
 426               void *buf,
 427               char **emsg)
 428 {
 429   struct GNUNET_FS_PublishContext *pc = cls;
 430   struct GNUNET_FS_FileInformation *p;
 431   size_t pt_size;
 432   const char *dd;
 433
 434   p = pc->fi_pos;
 435   if (p->is_directory)
 436     {
 437       pt_size = GNUNET_MIN(max,
 438                            p->data.dir.dir_size - offset);
 439       dd = p->data.dir.dir_data;
 440       memcpy (buf,
 441               &dd[offset],
 442               pt_size);
 443     }
 444   else
 445     {
 446       pt_size = GNUNET_MIN(max,
 447                            p->data.file.file_size - offset);
 448       if (pt_size == 0)
 449         return 0; /* calling reader with pt_size==0
 450                      might free buf, so don't! */
 451       if (pt_size !=
 452           p->data.file.reader (p->data.file.reader_cls,
 453                                offset,
 454                                pt_size,
 455                                buf,
 456                                emsg))
 457         return 0;
 458     }
 459   return pt_size;
 460 }
 461
 462
 463 /**
 464  * The tree encoder has finished processing a
 465  * file.   Call it's finish method and deal with
 466  * the final result.
 467  *
 468  * @param cls our publishing context
 469  * @param tc scheduler's task context (not used)
 470  */
 471 static void
 472 encode_cont (void *cls,
 473              const struct GNUNET_SCHEDULER_TaskContext *tc)
 474 {
 475   struct GNUNET_FS_PublishContext *pc = cls;
 476   struct GNUNET_FS_FileInformation *p;
 477   struct GNUNET_FS_ProgressInfo pi;
 478   char *emsg;
 479   uint64_t flen;
 480
 481   p = pc->fi_pos;
 482   GNUNET_FS_tree_encoder_finish (p->te,
 483                                  &p->chk_uri,
 484                                  &emsg);
 485   p->te = NULL;
 486   if (NULL != emsg)
 487     {
 488 #if DEBUG_PUBLISH
 489       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 490                   "Error during tree walk: %s\n",
 491                   emsg);
 492 #endif
 493       GNUNET_asprintf (&p->emsg,
 494                        _("Publishing failed: %s"),
 495                        emsg);
 496       GNUNET_free (emsg);
 497       pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
 498       pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
 499       pi.value.publish.specifics.error.message = p->emsg;
 500       p->client_info =  GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
 501     }
 502 #if DEBUG_PUBLISH
 503   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 504               "Finished with tree encoder\n");
 505 #endif
 506   /* final progress event */
 507   flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
 508   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 509   pi.value.publish.specifics.progress.data = NULL;
 510   pi.value.publish.specifics.progress.offset = flen;
 511   pi.value.publish.specifics.progress.data_len = 0;
 512   pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
 513   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen);
 514
 515   /* continue with main */
 516   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 517   pc->upload_task
 518     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 519                                           &GNUNET_FS_publish_main_,
 520                                           pc);
 521 }
 522
 523
 524 /**
 525  * Function called asking for the current (encoded)
 526  * block to be processed.  After processing the
 527  * client should either call "GNUNET_FS_tree_encode_next"
 528  * or (on error) "GNUNET_FS_tree_encode_finish".
 529  *
 530  * @param cls closure
 531  * @param chk content hash key for the block
 532  * @param offset offset of the block in the file
 533  * @param depth depth of the block in the file, 0 for DBLOCK
 534  * @param type type of the block (IBLOCK or DBLOCK)
 535  * @param block the (encrypted) block
 536  * @param block_size size of block (in bytes)
 537  */
 538 static void
 539 block_proc (void *cls,
 540             const struct ContentHashKey *chk,
 541             uint64_t offset,
 542             unsigned int depth,
 543             enum GNUNET_BLOCK_Type type,
 544             const void *block,
 545             uint16_t block_size)
 546 {
 547   struct GNUNET_FS_PublishContext *pc = cls;
 548   struct GNUNET_FS_FileInformation *p;
 549   struct PutContCtx * dpc_cls;
 550   struct OnDemandBlock odb;
 551
 552   p = pc->fi_pos;
 553   if (NULL == pc->dsh)
 554     {
 555 #if DEBUG_PUBLISH
 556       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 557                   "Waiting for datastore connection\n");
 558 #endif
 559       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
 560       pc->upload_task
 561         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
 562                                               &GNUNET_FS_publish_main_,
 563                                               pc);
 564       return;
 565     }
 566
 567   GNUNET_assert (GNUNET_NO == pc->in_network_wait);
 568   pc->in_network_wait = GNUNET_YES;
 569   dpc_cls = GNUNET_malloc(sizeof(struct PutContCtx));
 570   dpc_cls->cont = &GNUNET_FS_publish_main_;
 571   dpc_cls->cont_cls = pc;
 572   dpc_cls->pc = pc;
 573   dpc_cls->p = p;
 574   if ( (! p->is_directory) &&
 575        (GNUNET_YES == p->data.file.do_index) &&
 576        (type == GNUNET_BLOCK_TYPE_FS_DBLOCK) )
 577     {
 578 #if DEBUG_PUBLISH
 579       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 580                   "Indexing block `%s' for offset %llu with index size %u\n",
 581                   GNUNET_h2s (&chk->query),
 582                   (unsigned long long) offset,
 583                   sizeof (struct OnDemandBlock));
 584 #endif
 585       odb.offset = GNUNET_htonll (offset);
 586       odb.file_id = p->data.file.file_id;
 587       GNUNET_assert (pc->qre == NULL);
 588       pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 589                                       (p->is_directory) ? 0 : pc->rid,
 590                                       &chk->query,
 591                                       sizeof (struct OnDemandBlock),
 592                                       &odb,
 593                                       GNUNET_BLOCK_TYPE_FS_ONDEMAND,
 594                                       p->bo.content_priority,
 595                                       p->bo.anonymity_level,
 596                                       p->bo.replication_level,
 597                                       p->bo.expiration_time,
 598                                       -2, 1,
 599                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 600                                       &ds_put_cont,
 601                                       dpc_cls);
 602       return;
 603     }
 604 #if DEBUG_PUBLISH
 605   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 606               "Publishing block `%s' for offset %llu with size %u\n",
 607               GNUNET_h2s (&chk->query),
 608               (unsigned long long) offset,
 609               (unsigned int) block_size);
 610 #endif
 611   GNUNET_assert (pc->qre == NULL);
 612   pc->qre = GNUNET_DATASTORE_put (pc->dsh,
 613                                   (p->is_directory) ? 0 : pc->rid,
 614                                   &chk->query,
 615                                   block_size,
 616                                   block,
 617                                   type,
 618                                   p->bo.content_priority,
 619                                   p->bo.anonymity_level,
 620                                   p->bo.replication_level,
 621                                   p->bo.expiration_time,
 622                                   -2, 1,
 623                                   GNUNET_CONSTANTS_SERVICE_TIMEOUT,
 624                                   &ds_put_cont,
 625                                   dpc_cls);
 626 }
 627
 628
 629 /**
 630  * Function called with information about our
 631  * progress in computing the tree encoding.
 632  *
 633  * @param cls closure
 634  * @param offset where are we in the file
 635  * @param pt_block plaintext of the currently processed block
 636  * @param pt_size size of pt_block
 637  * @param depth depth of the block in the tree, 0 for DBLOCK
 638  */
 639 static void
 640 progress_proc (void *cls,
 641                uint64_t offset,
 642                const void *pt_block,
 643                size_t pt_size,
 644                unsigned int depth)
 645 {
 646   struct GNUNET_FS_PublishContext *pc = cls;
 647   struct GNUNET_FS_FileInformation *p;
 648   struct GNUNET_FS_ProgressInfo pi;
 649
 650   p = pc->fi_pos;
 651   pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
 652   pi.value.publish.specifics.progress.data = pt_block;
 653   pi.value.publish.specifics.progress.offset = offset;
 654   pi.value.publish.specifics.progress.data_len = pt_size;
 655   pi.value.publish.specifics.progress.depth = depth;
 656   p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset);
 657 }
 658
 659
 660 /**
 661  * We are uploading a file or directory; load (if necessary) the next
 662  * block into memory, encrypt it and send it to the FS service.  Then
 663  * continue with the main task.
 664  *
 665  * @param pc overall upload data
 666  */
 667 static void
 668 publish_content (struct GNUNET_FS_PublishContext *pc)
 669 {
 670   struct GNUNET_FS_FileInformation *p;
 671   char *emsg;
 672   struct GNUNET_FS_DirectoryBuilder *db;
 673   struct GNUNET_FS_FileInformation *dirpos;
 674   void *raw_data;
 675   uint64_t size;
 676
 677   p = pc->fi_pos;
 678   GNUNET_assert (p != NULL);
 679   if (NULL == p->te)
 680     {
 681       if (p->is_directory)
 682         {
 683 #if DEBUG_PUBLISH
 684           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 685                       "Creating directory\n");
 686 #endif
 687           db = GNUNET_FS_directory_builder_create (p->meta);
 688           dirpos = p->data.dir.entries;
 689           while (NULL != dirpos)
 690             {
 691               if (dirpos->is_directory)
 692                 {
 693                   raw_data = dirpos->data.dir.dir_data;
 694                   dirpos->data.dir.dir_data = NULL;
 695                 }
 696               else
 697                 {
 698                   raw_data = NULL;
 699                   if ( (dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
 700                        (dirpos->data.file.file_size > 0) )
 701                     {
 702                       raw_data = GNUNET_malloc (dirpos->data.file.file_size);
 703                       emsg = NULL;
 704                       if (dirpos->data.file.file_size !=
 705                           dirpos->data.file.reader (dirpos->data.file.reader_cls,
 706                                                     0,
 707                                                     dirpos->data.file.file_size,
 708                                                     raw_data,
 709                                                     &emsg))
 710                         {
 711                           GNUNET_free_non_null (emsg);
 712                           GNUNET_free (raw_data);
 713                           raw_data = NULL;
 714                         }
 715                     }
 716                 }
 717               GNUNET_FS_directory_builder_add (db,
 718                                                dirpos->chk_uri,
 719                                                dirpos->meta,
 720                                                raw_data);
 721               GNUNET_free_non_null (raw_data);
 722               dirpos = dirpos->next;
 723             }
 724           GNUNET_FS_directory_builder_finish (db,
 725                                               &p->data.dir.dir_size,
 726                                               &p->data.dir.dir_data);
 727           GNUNET_FS_file_information_sync_ (p);
 728         }
 729       size = (p->is_directory)
 730         ? p->data.dir.dir_size
 731         : p->data.file.file_size;
 732 #if DEBUG_PUBLISH
 733       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 734                   "Creating tree encoder\n");
 735 #endif
 736       p->te = GNUNET_FS_tree_encoder_create (pc->h,
 737                                              size,
 738                                              pc,
 739                                              &block_reader,
 740                                              &block_proc,
 741                                              &progress_proc,
 742                                              &encode_cont);
 743
 744     }
 745 #if DEBUG_PUBLISH
 746   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 747               "Processing next block from tree\n");
 748 #endif
 749   GNUNET_FS_tree_encoder_next (p->te);
 750 }
 751
 752
 753 /**
 754  * Process the response (or lack thereof) from
 755  * the "fs" service to our 'start index' request.
 756  *
 757  * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_)
 758  * @param msg the response we got
 759  */
 760 static void
 761 process_index_start_response (void *cls,
 762                               const struct GNUNET_MessageHeader *msg)
 763 {
 764   struct GNUNET_FS_PublishContext *pc = cls;
 765   struct GNUNET_FS_FileInformation *p;
 766   const char *emsg;
 767   uint16_t msize;
 768
 769   GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO);
 770   pc->client = NULL;
 771   p = pc->fi_pos;
 772   if (msg == NULL)
 773     {
 774       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 775                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 776                   p->filename,
 777                   _("timeout on index-start request to `fs' service"));
 778       p->data.file.do_index = GNUNET_NO;
 779       GNUNET_FS_file_information_sync_ (p);
 780       publish_content (pc);
 781       return;
 782     }
 783   if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK)
 784     {
 785       msize = ntohs (msg->size);
 786       emsg = (const char *) &msg[1];
 787       if ( (msize <= sizeof (struct GNUNET_MessageHeader)) ||
 788            (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0') )
 789         emsg = gettext_noop ("unknown error");
 790       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 791                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 792                   p->filename,
 793                   gettext (emsg));
 794       p->data.file.do_index = GNUNET_NO;
 795       GNUNET_FS_file_information_sync_ (p);
 796       publish_content (pc);
 797       return;
 798     }
 799   p->data.file.index_start_confirmed = GNUNET_YES;
 800   /* success! continue with indexing */
 801   GNUNET_FS_file_information_sync_ (p);
 802   publish_content (pc);
 803 }
 804
 805
 806 /**
 807  * Function called once the hash computation over an
 808  * indexed file has completed.
 809  *
 810  * @param cls closure, our publishing context
 811  * @param res resulting hash, NULL on error
 812  */
 813 static void
 814 hash_for_index_cb (void *cls,
 815                    const GNUNET_HashCode *
 816                    res)
 817 {
 818   struct GNUNET_FS_PublishContext *pc = cls;
 819   struct GNUNET_FS_FileInformation *p;
 820   struct IndexStartMessage *ism;
 821   size_t slen;
 822   struct GNUNET_CLIENT_Connection *client;
 823   uint64_t dev;
 824   uint64_t ino;
 825   char *fn;
 826
 827   pc->fhc = NULL;
 828   p = pc->fi_pos;
 829   if (NULL == res)
 830     {
 831       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 832                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 833                   p->filename,
 834                   _("failed to compute hash"));
 835       p->data.file.do_index = GNUNET_NO;
 836       GNUNET_FS_file_information_sync_ (p);
 837       publish_content (pc);
 838       return;
 839     }
 840   if (GNUNET_YES == p->data.file.index_start_confirmed)
 841     {
 842       publish_content (pc);
 843       return;
 844     }
 845   fn = GNUNET_STRINGS_filename_expand (p->filename);
 846   GNUNET_assert (fn != NULL);
 847   slen = strlen (fn) + 1;
 848   if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
 849     {
 850       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 851                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 852                   fn,
 853                   _("filename too long"));
 854       GNUNET_free (fn);
 855       p->data.file.do_index = GNUNET_NO;
 856       GNUNET_FS_file_information_sync_ (p);
 857       publish_content (pc);
 858       return;
 859     }
 860 #if DEBUG_PUBLISH
 861   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 862               "Hash of indexed file `%s' is `%s'\n",
 863               p->filename,
 864               GNUNET_h2s (res));
 865 #endif
 866   if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
 867     {
 868       p->data.file.file_id = *res;
 869       p->data.file.have_hash = GNUNET_YES;
 870       p->data.file.index_start_confirmed = GNUNET_YES;
 871       GNUNET_FS_file_information_sync_ (p);
 872       publish_content (pc);
 873       GNUNET_free (fn);
 874       return;
 875     }
 876   client = GNUNET_CLIENT_connect ("fs",
 877                                   pc->h->cfg);
 878   if (NULL == client)
 879     {
 880       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 881                   _("Can not index file `%s': %s.  Will try to insert instead.\n"),
 882                   p->filename,
 883                   _("could not connect to `fs' service"));
 884       p->data.file.do_index = GNUNET_NO;
 885       publish_content (pc);
 886       GNUNET_free (fn);
 887       return;
 888     }
 889   if (p->data.file.have_hash != GNUNET_YES)
 890     {
 891       p->data.file.file_id = *res;
 892       p->data.file.have_hash = GNUNET_YES;
 893       GNUNET_FS_file_information_sync_ (p);
 894     }
 895   ism = GNUNET_malloc (sizeof(struct IndexStartMessage) +
 896                        slen);
 897   ism->header.size = htons(sizeof(struct IndexStartMessage) +
 898                            slen);
 899   ism->header.type = htons(GNUNET_MESSAGE_TYPE_FS_INDEX_START);
 900   if (GNUNET_OK ==
 901       GNUNET_DISK_file_get_identifiers (p->filename,
 902                                         &dev,
 903                                         &ino))
 904     {
 905       ism->device = GNUNET_htonll (dev);
 906       ism->inode = GNUNET_htonll(ino);
 907     }
 908 #if DEBUG_PUBLISH
 909   else
 910     {
 911       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 912                   _("Failed to get file identifiers for `%s'\n"),
 913                   p->filename);
 914     }
 915 #endif
 916   ism->file_id = *res;
 917   memcpy (&ism[1],
 918           fn,
 919           slen);
 920   GNUNET_free (fn);
 921   pc->client = client;
 922   GNUNET_break (GNUNET_YES ==
 923                 GNUNET_CLIENT_transmit_and_get_response (client,
 924                                                          &ism->header,
 925                                                          GNUNET_TIME_UNIT_FOREVER_REL,
 926                                                          GNUNET_YES,
 927                                                          &process_index_start_response,
 928                                                          pc));
 929   GNUNET_free (ism);
 930 }
 931
 932
 933 /**
 934  * Main function that performs the upload.
 935  *
 936  * @param cls "struct GNUNET_FS_PublishContext" identifies the upload
 937  * @param tc task context
 938  */
 939 void
 940 GNUNET_FS_publish_main_ (void *cls,
 941                          const struct GNUNET_SCHEDULER_TaskContext *tc)
 942 {
 943   struct GNUNET_FS_PublishContext *pc = cls;
 944   struct GNUNET_FS_ProgressInfo pi;
 945   struct GNUNET_FS_FileInformation *p;
 946   struct GNUNET_FS_Uri *loc;
 947   char *fn;
 948
 949   pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
 950   p = pc->fi_pos;
 951   if (NULL == p)
 952     {
 953 #if DEBUG_PUBLISH
 954       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 955                   "Publishing complete, now publishing SKS and KSK blocks.\n");
 956 #endif
 957       /* upload of entire hierarchy complete,
 958          publish namespace entries */
 959       GNUNET_FS_publish_sync_ (pc);
 960       publish_sblock (pc);
 961       return;
 962     }
 963   /* find starting position */
 964   while ( (p->is_directory) &&
 965           (NULL != p->data.dir.entries) &&
 966           (NULL == p->emsg) &&
 967           (NULL == p->data.dir.entries->chk_uri) )
 968     {
 969       p = p->data.dir.entries;
 970       pc->fi_pos = p;
 971       GNUNET_FS_publish_sync_ (pc);
 972     }
 973   /* abort on error */
 974   if (NULL != p->emsg)
 975     {
 976 #if DEBUG_PUBLISH
 977       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 978                   "Error uploading: %s\n",
 979                   p->emsg);
 980 #endif
 981       /* error with current file, abort all
 982          related files as well! */
 983       while (NULL != p->dir)
 984         {
 985           fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
 986                                                        EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
 987           p = p->dir;
 988           if (fn != NULL)
 989             {
 990               GNUNET_asprintf (&p->emsg,
 991                                _("Recursive upload failed at `%s': %s"),
 992                                fn,
 993                                p->emsg);
 994               GNUNET_free (fn);
 995             }
 996           else
 997             {
 998               GNUNET_asprintf (&p->emsg,
 999                                _("Recursive upload failed: %s"),
1000                                p->emsg);
1001             }
1002           pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
1003           pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
1004           pi.value.publish.specifics.error.message = p->emsg;
1005           p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
1006         }
1007       pc->all_done = GNUNET_YES;
1008       GNUNET_FS_publish_sync_ (pc);
1009       return;
1010     }
1011   /* handle completion */
1012   if (NULL != p->chk_uri)
1013     {
1014 #if DEBUG_PUBLISH
1015       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1016                   "File upload complete, now publishing KSK blocks.\n");
1017 #endif
1018       if (0 == p->bo.anonymity_level)
1019         {
1020           /* zero anonymity, box CHK URI in LOC URI */
1021           loc = GNUNET_FS_uri_loc_create (p->chk_uri,
1022                                           pc->h->cfg,
1023                                           p->bo.expiration_time);
1024           GNUNET_FS_uri_destroy (p->chk_uri);
1025           p->chk_uri = loc;
1026         }
1027       GNUNET_FS_publish_sync_ (pc);
1028       /* upload of "p" complete, publish KBlocks! */
1029       if (p->keywords != NULL)
1030         {
1031           GNUNET_FS_publish_ksk (pc->h,
1032                                  p->keywords,
1033                                  p->meta,
1034                                  p->chk_uri,
1035                                  &p->bo,
1036                                  pc->options,
1037                                  &publish_kblocks_cont,
1038                                  pc);
1039         }
1040       else
1041         {
1042           publish_kblocks_cont (pc,
1043                                 p->chk_uri,
1044                                 NULL);
1045         }
1046       return;
1047     }
1048   if ( (!p->is_directory) &&
1049        (p->data.file.do_index) )
1050     {
1051       if (NULL == p->filename)
1052         {
1053           p->data.file.do_index = GNUNET_NO;
1054           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1055                       _("Can not index file `%s': %s.  Will try to insert instead.\n"),
1056                       "<no-name>",
1057                       _("needs to be an actual file"));
1058           GNUNET_FS_file_information_sync_ (p);
1059           publish_content (pc);
1060           return;
1061         }
1062       if (p->data.file.have_hash)
1063         {
1064           hash_for_index_cb (pc,
1065                              &p->data.file.file_id);
1066         }
1067       else
1068         {
1069           p->start_time = GNUNET_TIME_absolute_get ();
1070           pc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
1071                                              p->filename,
1072                                              HASHING_BLOCKSIZE,
1073                                              &hash_for_index_cb,
1074                                              pc);
1075         }
1076       return;
1077     }
1078   publish_content (pc);
1079 }
1080
1081
1082 /**
1083  * Signal the FS's progress function that we are starting
1084  * an upload.
1085  *
1086  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1087  * @param fi the entry in the publish-structure
1088  * @param length length of the file or directory
1089  * @param meta metadata for the file or directory (can be modified)
1090  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1091  * @param bo block options
1092  * @param do_index should we index?
1093  * @param client_info pointer to client context set upon creation (can be modified)
1094  * @return GNUNET_OK to continue (always)
1095  */
1096 static int
1097 fip_signal_start(void *cls,
1098                  struct GNUNET_FS_FileInformation *fi,
1099                  uint64_t length,
1100                  struct GNUNET_CONTAINER_MetaData *meta,
1101                  struct GNUNET_FS_Uri **uri,
1102                  struct GNUNET_FS_BlockOptions *bo,
1103                  int *do_index,
1104                  void **client_info)
1105 {
1106   struct GNUNET_FS_PublishContext *pc = cls;
1107   struct GNUNET_FS_ProgressInfo pi;
1108   unsigned int kc;
1109   uint64_t left;
1110
1111 #if DEBUG_PUBLISH
1112   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1113               "Starting publish operation\n");
1114 #endif
1115   if (*do_index)
1116     {
1117       /* space for on-demand blocks */
1118       pc->reserve_space += ((length + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock);
1119     }
1120   else
1121     {
1122       /* space for DBlocks */
1123       pc->reserve_space += length;
1124     }
1125   /* entries for IBlocks and DBlocks, space for IBlocks */
1126   left = length;
1127   while (1)
1128     {
1129       left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1130       pc->reserve_entries += left;
1131       if (left <= 1)
1132         break;
1133       left = left * sizeof (struct ContentHashKey);
1134       pc->reserve_space += left;
1135     }
1136   pc->reserve_entries++;
1137   /* entries and space for keywords */
1138   if (NULL != *uri)
1139     {
1140       kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
1141       pc->reserve_entries += kc;
1142       pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
1143     }
1144   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
1145   *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
1146   GNUNET_FS_file_information_sync_ (fi);
1147   return GNUNET_OK;
1148 }
1149
1150
1151 /**
1152  * Signal the FS's progress function that we are suspending
1153  * an upload.
1154  *
1155  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1156  * @param fi the entry in the publish-structure
1157  * @param length length of the file or directory
1158  * @param meta metadata for the file or directory (can be modified)
1159  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1160  * @param bo block options
1161  * @param do_index should we index?
1162  * @param client_info pointer to client context set upon creation (can be modified)
1163  * @return GNUNET_OK to continue (always)
1164  */
1165 static int
1166 fip_signal_suspend(void *cls,
1167                    struct GNUNET_FS_FileInformation *fi,
1168                    uint64_t length,
1169                    struct GNUNET_CONTAINER_MetaData *meta,
1170                    struct GNUNET_FS_Uri **uri,
1171                    struct GNUNET_FS_BlockOptions *bo,
1172                    int *do_index,
1173                    void **client_info)
1174 {
1175   struct GNUNET_FS_PublishContext*pc = cls;
1176   struct GNUNET_FS_ProgressInfo pi;
1177   uint64_t off;
1178
1179 #if DEBUG_PUBLISH
1180   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1181               "Suspending publish operation\n");
1182 #endif
1183   GNUNET_free_non_null (fi->serialization);
1184   fi->serialization = NULL;
1185   off = (fi->chk_uri == NULL) ? 0 : length;
1186   pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
1187   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1188   *client_info = NULL;
1189   if (NULL != pc->qre)
1190     {
1191       GNUNET_DATASTORE_cancel (pc->qre);
1192       pc->qre = NULL;
1193     }
1194   if (NULL != pc->dsh)
1195     {
1196       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1197       pc->dsh = NULL;
1198     }
1199   pc->rid = 0;
1200   return GNUNET_OK;
1201 }
1202
1203
1204 /**
1205  * Create SUSPEND event for the given publish operation
1206  * and then clean up our state (without stop signal).
1207  *
1208  * @param cls the 'struct GNUNET_FS_PublishContext' to signal for
1209  */
1210 void
1211 GNUNET_FS_publish_signal_suspend_ (void *cls)
1212 {
1213   struct GNUNET_FS_PublishContext *pc = cls;
1214
1215   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1216     {
1217       GNUNET_SCHEDULER_cancel (pc->upload_task);
1218       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1219     }
1220   GNUNET_FS_file_information_inspect (pc->fi,
1221                                       &fip_signal_suspend,
1222                                       pc);
1223   GNUNET_FS_end_top (pc->h, pc->top);
1224   pc->top = NULL;
1225   publish_cleanup (pc, NULL);
1226 }
1227
1228
1229 /**
1230  * We have gotten a reply for our space reservation request.
1231  * Either fail (insufficient space) or start publishing for good.
1232  *
1233  * @param cls the 'struct GNUNET_FS_PublishContext*'
1234  * @param success positive reservation ID on success
1235  * @param msg error message on error, otherwise NULL
1236  */
1237 static void
1238 finish_reserve (void *cls,
1239                 int success,
1240                 const char *msg)
1241 {
1242   struct GNUNET_FS_PublishContext *pc = cls;
1243
1244   pc->qre = NULL;
1245 #if DEBUG_PUBLISH
1246   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1247               "Reservation complete (%d)!\n",
1248               success);
1249 #endif
1250   if ( (msg != NULL) ||
1251        (success <= 0) )
1252     {
1253       GNUNET_asprintf (&pc->fi->emsg,
1254                        _("Insufficient space for publishing: %s"),
1255                        msg);
1256       signal_publish_error (pc->fi,
1257                             pc,
1258                             pc->fi->emsg);
1259       return;
1260     }
1261   pc->rid = success;
1262   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task);
1263   pc->upload_task
1264     = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1265                                           &GNUNET_FS_publish_main_,
1266                                           pc);
1267 }
1268
1269
1270 /**
1271  * Publish a file or directory.
1272  *
1273  * @param h handle to the file sharing subsystem
1274  * @param fi information about the file or directory structure to publish
1275  * @param namespace namespace to publish the file in, NULL for no namespace
1276  * @param nid identifier to use for the publishd content in the namespace
1277  *        (can be NULL, must be NULL if namespace is NULL)
1278  * @param nuid update-identifier that will be used for future updates
1279  *        (can be NULL, must be NULL if namespace or nid is NULL)
1280  * @param options options for the publication
1281  * @return context that can be used to control the publish operation
1282  */
1283 struct GNUNET_FS_PublishContext *
1284 GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
1285                          struct GNUNET_FS_FileInformation *fi,
1286                          struct GNUNET_FS_Namespace *namespace,
1287                          const char *nid,
1288                          const char *nuid,
1289                          enum GNUNET_FS_PublishOptions options)
1290 {
1291   struct GNUNET_FS_PublishContext *ret;
1292   struct GNUNET_DATASTORE_Handle *dsh;
1293
1294   GNUNET_assert (NULL != h);
1295   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1296     {
1297       dsh = GNUNET_DATASTORE_connect (h->cfg);
1298       if (NULL == dsh)
1299         return NULL;
1300     }
1301   else
1302     {
1303       dsh = NULL;
1304     }
1305   ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
1306   ret->dsh = dsh;
1307   ret->h = h;
1308   ret->fi = fi;
1309   ret->namespace = namespace;
1310   ret->options = options;
1311   if (namespace != NULL)
1312     {
1313       namespace->rc++;
1314       GNUNET_assert (NULL != nid);
1315       ret->nid = GNUNET_strdup (nid);
1316       if (NULL != nuid)
1317         ret->nuid = GNUNET_strdup (nuid);
1318     }
1319   /* signal start */
1320   GNUNET_FS_file_information_inspect (ret->fi,
1321                                       &fip_signal_start,
1322                                       ret);
1323   ret->fi_pos = ret->fi;
1324   ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
1325   GNUNET_FS_publish_sync_ (ret);
1326   if (NULL != ret->dsh)
1327     {
1328       GNUNET_assert (NULL == ret->qre);
1329       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1330                   _("Reserving space for %u entries and %llu bytes for publication\n"),
1331                   (unsigned int) ret->reserve_entries,
1332                   (unsigned long long) ret->reserve_space);
1333       ret->qre = GNUNET_DATASTORE_reserve (ret->dsh,
1334                                            ret->reserve_space,
1335                                            ret->reserve_entries,
1336                                            UINT_MAX,
1337                                            UINT_MAX,
1338                                            GNUNET_TIME_UNIT_FOREVER_REL,
1339                                            &finish_reserve,
1340                                            ret);
1341     }
1342   else
1343     {
1344       GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == ret->upload_task);
1345       ret->upload_task
1346         = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
1347                                               &GNUNET_FS_publish_main_,
1348                                               ret);
1349     }
1350   return ret;
1351 }
1352
1353
1354 /**
1355  * Signal the FS's progress function that we are stopping
1356  * an upload.
1357  *
1358  * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
1359  * @param fi the entry in the publish-structure
1360  * @param length length of the file or directory
1361  * @param meta metadata for the file or directory (can be modified)
1362  * @param uri pointer to the keywords that will be used for this entry (can be modified)
1363  * @param bo block options (can be modified)
1364  * @param do_index should we index?
1365  * @param client_info pointer to client context set upon creation (can be modified)
1366  * @return GNUNET_OK to continue (always)
1367  */
1368 static int
1369 fip_signal_stop(void *cls,
1370                 struct GNUNET_FS_FileInformation *fi,
1371                 uint64_t length,
1372                 struct GNUNET_CONTAINER_MetaData *meta,
1373                 struct GNUNET_FS_Uri **uri,
1374                 struct GNUNET_FS_BlockOptions *bo,
1375                 int *do_index,
1376                 void **client_info)
1377 {
1378   struct GNUNET_FS_PublishContext*pc = cls;
1379   struct GNUNET_FS_ProgressInfo pi;
1380   uint64_t off;
1381
1382   if (fi->serialization != NULL)
1383     {
1384       GNUNET_FS_remove_sync_file_ (pc->h,
1385                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
1386                                    fi->serialization);
1387       GNUNET_free (fi->serialization);
1388       fi->serialization = NULL;
1389     }
1390   off = (fi->chk_uri == NULL) ? 0 : length;
1391   pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
1392   GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
1393   *client_info = NULL;
1394   return GNUNET_OK;
1395 }
1396
1397
1398 /**
1399  * Stop an upload.  Will abort incomplete uploads (but
1400  * not remove blocks that have already been publishd) or
1401  * simply clean up the state for completed uploads.
1402  * Must NOT be called from within the event callback!
1403  *
1404  * @param pc context for the upload to stop
1405  */
1406 void
1407 GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
1408 {
1409 #if DEBUG_PUBLISH
1410   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1411               "Publish stop called\n");
1412 #endif
1413   GNUNET_FS_end_top (pc->h, pc->top);
1414   if (NULL != pc->qre)
1415     {
1416       GNUNET_DATASTORE_cancel (pc->qre);
1417       pc->qre = NULL;
1418     }
1419   if (NULL != pc->dsh)
1420     {
1421       GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
1422       pc->dsh = NULL;
1423     }
1424   if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task)
1425     {
1426       GNUNET_SCHEDULER_cancel (pc->upload_task);
1427       pc->upload_task = GNUNET_SCHEDULER_NO_TASK;
1428     }
1429   if (pc->serialization != NULL)
1430     {
1431       GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, pc->serialization);
1432       GNUNET_free (pc->serialization);
1433       pc->serialization = NULL;
1434     }
1435   GNUNET_FS_file_information_inspect (pc->fi,
1436                                       &fip_signal_stop,
1437                                       pc);
1438   if (GNUNET_YES == pc->in_network_wait)
1439     {
1440       pc->in_network_wait = GNUNET_SYSERR;
1441       return;
1442     }
1443   publish_cleanup (pc, NULL);
1444 }
1445
1446
1447 /**
1448  * Context for the KSK publication.
1449  */
1450 struct PublishKskContext
1451 {
1452
1453   /**
1454    * Keywords to use.
1455    */
1456   struct GNUNET_FS_Uri *ksk_uri;
1457
1458   /**
1459    * Global FS context.
1460    */
1461   struct GNUNET_FS_Handle *h;
1462
1463   /**
1464    * The master block that we are sending
1465    * (in plaintext), has "mdsize+slen" more
1466    * bytes than the struct would suggest.
1467    */
1468   struct KBlock *kb;
1469
1470   /**
1471    * Buffer of the same size as "kb" for
1472    * the encrypted version.
1473    */
1474   struct KBlock *cpy;
1475
1476   /**
1477    * Handle to the datastore, NULL if we are just
1478    * simulating.
1479    */
1480   struct GNUNET_DATASTORE_Handle *dsh;
1481
1482   /**
1483    * Handle to datastore PUT request.
1484    */
1485   struct GNUNET_DATASTORE_QueueEntry *qre;
1486
1487   /**
1488    * Function to call once we're done.
1489    */
1490   GNUNET_FS_PublishContinuation cont;
1491
1492   /**
1493    * Closure for cont.
1494    */
1495   void *cont_cls;
1496
1497   /**
1498    * When should the KBlocks expire?
1499    */
1500   struct GNUNET_FS_BlockOptions bo;
1501
1502   /**
1503    * Size of the serialized metadata.
1504    */
1505   ssize_t mdsize;
1506
1507   /**
1508    * Size of the (CHK) URI as a string.
1509    */
1510   size_t slen;
1511
1512   /**
1513    * Keyword that we are currently processing.
1514    */
1515   unsigned int i;
1516
1517 };
1518
1519
1520 /**
1521  * Continuation of "GNUNET_FS_publish_ksk" that performs
1522  * the actual publishing operation (iterating over all
1523  * of the keywords).
1524  *
1525  * @param cls closure of type "struct PublishKskContext*"
1526  * @param tc unused
1527  */
1528 static void
1529 publish_ksk_cont (void *cls,
1530                   const struct GNUNET_SCHEDULER_TaskContext *tc);
1531
1532
1533 /**
1534  * Function called by the datastore API with
1535  * the result from the PUT request.
1536  *
1537  * @param cls closure of type "struct PublishKskContext*"
1538  * @param success GNUNET_OK on success
1539  * @param msg error message (or NULL)
1540  */
1541 static void
1542 kb_put_cont (void *cls,
1543              int success,
1544              const char *msg)
1545 {
1546   struct PublishKskContext *pkc = cls;
1547
1548   pkc->qre = NULL;
1549   if (GNUNET_OK != success)
1550     {
1551 #if DEBUG_PUBLISH
1552       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1553                   "KB PUT operation complete\n");
1554 #endif
1555       if (NULL != pkc->dsh)
1556         {
1557           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1558           pkc->dsh = NULL;
1559         }
1560       GNUNET_free (pkc->cpy);
1561       GNUNET_free (pkc->kb);
1562       pkc->cont (pkc->cont_cls,
1563                  NULL,
1564                  msg);
1565       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1566       GNUNET_free (pkc);
1567       return;
1568     }
1569   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1570                                      pkc,
1571                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1572 }
1573
1574
1575 /**
1576  * Continuation of "GNUNET_FS_publish_ksk" that performs the actual
1577  * publishing operation (iterating over all of the keywords).
1578  *
1579  * @param cls closure of type "struct PublishKskContext*"
1580  * @param tc unused
1581  */
1582 static void
1583 publish_ksk_cont (void *cls,
1584                   const struct GNUNET_SCHEDULER_TaskContext *tc)
1585 {
1586   struct PublishKskContext *pkc = cls;
1587   const char *keyword;
1588   GNUNET_HashCode key;
1589   GNUNET_HashCode query;
1590   struct GNUNET_CRYPTO_AesSessionKey skey;
1591   struct GNUNET_CRYPTO_AesInitializationVector iv;
1592   struct GNUNET_CRYPTO_RsaPrivateKey *pk;
1593
1594
1595   if ( (pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
1596        (NULL == pkc->dsh) )
1597     {
1598 #if DEBUG_PUBLISH
1599       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1600                   "KSK PUT operation complete\n");
1601 #endif
1602       if (NULL != pkc->dsh)
1603         {
1604           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1605           pkc->dsh = NULL;
1606         }
1607       GNUNET_free (pkc->cpy);
1608       GNUNET_free (pkc->kb);
1609       pkc->cont (pkc->cont_cls,
1610                  pkc->ksk_uri,
1611                  NULL);
1612       GNUNET_FS_uri_destroy (pkc->ksk_uri);
1613       GNUNET_free (pkc);
1614       return;
1615     }
1616   keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
1617 #if DEBUG_PUBLISH
1618   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1619               "Publishing under keyword `%s'\n",
1620               keyword);
1621 #endif
1622   /* first character of keyword indicates if it is
1623      mandatory or not -- ignore for hashing */
1624   GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key);
1625   GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv);
1626   GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1],
1627                              pkc->slen + pkc->mdsize,
1628                              &skey,
1629                              &iv,
1630                              &pkc->cpy[1]);
1631   pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key);
1632   GNUNET_assert (NULL != pk);
1633   GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace);
1634   GNUNET_CRYPTO_hash (&pkc->cpy->keyspace,
1635                       sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
1636                       &query);
1637   GNUNET_assert (GNUNET_OK ==
1638                  GNUNET_CRYPTO_rsa_sign (pk,
1639                                          &pkc->cpy->purpose,
1640                                          &pkc->cpy->signature));
1641   GNUNET_CRYPTO_rsa_key_free (pk);
1642   pkc->qre = GNUNET_DATASTORE_put (pkc->dsh,
1643                                    0,
1644                                    &query,
1645                                    pkc->mdsize +
1646                                    sizeof (struct KBlock) +
1647                                    pkc->slen,
1648                                    pkc->cpy,
1649                                    GNUNET_BLOCK_TYPE_FS_KBLOCK,
1650                                    pkc->bo.content_priority,
1651                                    pkc->bo.anonymity_level,
1652                                    pkc->bo.replication_level,
1653                                    pkc->bo.expiration_time,
1654                                    -2, 1,
1655                                    GNUNET_CONSTANTS_SERVICE_TIMEOUT,
1656                                    &kb_put_cont,
1657                                    pkc);
1658 }
1659
1660
1661 /**
1662  * Publish a CHK under various keywords on GNUnet.
1663  *
1664  * @param h handle to the file sharing subsystem
1665  * @param ksk_uri keywords to use
1666  * @param meta metadata to use
1667  * @param uri URI to refer to in the KBlock
1668  * @param bo per-block options
1669  * @param options publication options
1670  * @param cont continuation
1671  * @param cont_cls closure for cont
1672  */
1673 void
1674 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1675                        const struct GNUNET_FS_Uri *ksk_uri,
1676                        const struct GNUNET_CONTAINER_MetaData *meta,
1677                        const struct GNUNET_FS_Uri *uri,
1678                        const struct GNUNET_FS_BlockOptions *bo,
1679                        enum GNUNET_FS_PublishOptions options,
1680                        GNUNET_FS_PublishContinuation cont,
1681                        void *cont_cls)
1682 {
1683   struct PublishKskContext *pkc;
1684   char *uris;
1685   size_t size;
1686   char *kbe;
1687   char *sptr;
1688
1689   GNUNET_assert (NULL != uri);
1690   pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1691   pkc->h = h;
1692   pkc->bo = *bo;
1693   pkc->cont = cont;
1694   pkc->cont_cls = cont_cls;
1695   if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1696     {
1697       pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
1698       if (pkc->dsh == NULL)
1699         {
1700           cont (cont_cls, NULL, _("Could not connect to datastore."));
1701           GNUNET_free (pkc);
1702           return;
1703         }
1704     }
1705   if (meta == NULL)
1706     pkc->mdsize = 0;
1707   else
1708     pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1709   GNUNET_assert (pkc->mdsize >= 0);
1710   uris = GNUNET_FS_uri_to_string (uri);
1711   pkc->slen = strlen (uris) + 1;
1712   size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen;
1713   if (size > MAX_KBLOCK_SIZE)
1714     {
1715       size = MAX_KBLOCK_SIZE;
1716       pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen;
1717     }
1718   pkc->kb = GNUNET_malloc (size);
1719   kbe = (char *) &pkc->kb[1];
1720   memcpy (kbe, uris, pkc->slen);
1721   GNUNET_free (uris);
1722   sptr = &kbe[pkc->slen];
1723   if (meta != NULL)
1724     pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1725                                                         &sptr,
1726                                                         pkc->mdsize,
1727                                                         GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1728   if (pkc->mdsize == -1)
1729     {
1730       GNUNET_break (0);
1731       GNUNET_free (pkc->kb);
1732       if (pkc->dsh != NULL)
1733         {
1734           GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
1735           pkc->dsh = NULL;
1736         }
1737       cont (cont_cls, NULL, _("Internal error."));
1738       GNUNET_free (pkc);
1739       return;
1740     }
1741   size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize;
1742
1743   pkc->cpy = GNUNET_malloc (size);
1744   pkc->cpy->purpose.size = htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) +
1745                                   sizeof(struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) +
1746                                   pkc->mdsize +
1747                                   pkc->slen);
1748   pkc->cpy->purpose.purpose = htonl(GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK);
1749   pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
1750   GNUNET_SCHEDULER_add_continuation (&publish_ksk_cont,
1751                                      pkc,
1752                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
1753 }
1754
1755
1756 /* end of fs_publish.c */