2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
23 * @brief main FS functions
24 * @author Christian Grothoff
28 #include "gnunet_util_lib.h"
29 #include "gnunet_fs_service.h"
34 * Start the given job (send signal, remove from pending queue, update
35 * counters and state).
37 * @param qe job to start
40 start_job (struct GNUNET_FS_QueueEntry *qe)
42 qe->client = GNUNET_CLIENT_connect (qe->h->sched, "fs", qe->h->cfg);
43 if (qe->client == NULL)
48 qe->start (qe->cls, qe->client);
50 qe->h->active_blocks += qe->blocks;
51 qe->start_time = GNUNET_TIME_absolute_get ();
52 GNUNET_CONTAINER_DLL_remove (qe->h->pending_head,
55 GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head,
63 * Stop the given job (send signal, remove from active queue, update
64 * counters and state).
66 * @param qe job to stop
69 stop_job (struct GNUNET_FS_QueueEntry *qe)
73 qe->h->active_downloads--;
74 qe->h->active_blocks -= qe->blocks;
75 qe->run_time = GNUNET_TIME_relative_add (qe->run_time,
76 GNUNET_TIME_absolute_get_duration (qe->start_time));
77 GNUNET_CONTAINER_DLL_remove (qe->h->running_head,
80 GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head,
88 * Process the jobs in the job queue, possibly starting some
89 * and stopping others.
91 * @param cls the 'struct GNUNET_FS_Handle'
92 * @param tc scheduler context
95 process_job_queue (void *cls,
96 const struct GNUNET_SCHEDULER_TaskContext *tc)
98 struct GNUNET_FS_Handle *h = cls;
99 struct GNUNET_FS_QueueEntry *qe;
100 struct GNUNET_FS_QueueEntry *next;
101 struct GNUNET_TIME_Relative run_time;
102 struct GNUNET_TIME_Relative restart_at;
103 struct GNUNET_TIME_Relative rst;
104 struct GNUNET_TIME_Absolute end_time;
106 h->queue_job = GNUNET_SCHEDULER_NO_TASK;
107 next = h->pending_head;
108 while (NULL != (qe = next))
111 if (h->running_head == NULL)
116 if ( (qe->blocks + h->active_blocks <= h->max_parallel_requests) &&
117 (h->active_downloads + 1 <= h->max_parallel_downloads) )
123 if (h->pending_head == NULL)
124 return; /* no need to stop anything */
125 restart_at = GNUNET_TIME_UNIT_FOREVER_REL;
126 next = h->running_head;
127 while (NULL != (qe = next))
130 /* FIXME: might be faster/simpler to do this calculation only once
131 when we start a job (OTOH, this would allow us to dynamically
132 and easily adjust qe->blocks over time, given the right API...) */
133 run_time = GNUNET_TIME_relative_multiply (h->avg_block_latency,
134 qe->blocks * qe->start_times);
135 end_time = GNUNET_TIME_absolute_add (qe->start_time,
137 rst = GNUNET_TIME_absolute_get_remaining (end_time);
138 restart_at = GNUNET_TIME_relative_min (rst, restart_at);
143 h->queue_job = GNUNET_SCHEDULER_add_delayed (h->sched,
151 * Add a job to the queue.
153 * @param h handle to the overall FS state
154 * @param start function to call to begin the job
155 * @param stop function to call to pause the job, or on dequeue (if the job was running)
156 * @param cls closure for start and stop
157 * @param blocks number of blocks this jobs uses
158 * @return queue handle
160 struct GNUNET_FS_QueueEntry *
161 GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h,
162 GNUNET_FS_QueueStart start,
163 GNUNET_FS_QueueStop stop,
167 struct GNUNET_FS_QueueEntry *qe;
169 qe = GNUNET_malloc (sizeof (struct GNUNET_FS_QueueEntry));
174 qe->queue_time = GNUNET_TIME_absolute_get ();
176 GNUNET_CONTAINER_DLL_insert_after (h->pending_head,
180 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
181 GNUNET_SCHEDULER_cancel (h->sched,
184 = GNUNET_SCHEDULER_add_now (h->sched,
192 * Dequeue a job from the queue.
193 * @param qh handle for the job
196 GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh)
198 struct GNUNET_FS_Handle *h;
201 if (qh->client != NULL)
203 GNUNET_CONTAINER_DLL_remove (h->pending_head,
207 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
208 GNUNET_SCHEDULER_cancel (h->sched,
211 = GNUNET_SCHEDULER_add_now (h->sched,
218 * Return the full filename where we would store state information
219 * (for serialization/deserialization).
221 * @param h master context
222 * @param ext component of the path
223 * @param ent entity identifier (or emtpy string for the directory)
224 * @return NULL on error
227 get_serialization_file_name (struct GNUNET_FS_Handle *h,
234 GNUNET_CONFIGURATION_get_value_filename (h->cfg,
239 GNUNET_asprintf (&ret,
247 GNUNET_free (basename);
253 * Return a read handle for deserialization.
255 * @param h master context
256 * @param ext component of the path
257 * @param ent entity identifier (or emtpy string for the directory)
258 * @return NULL on error
260 static struct GNUNET_BIO_ReadHandle *
261 get_read_handle (struct GNUNET_FS_Handle *h,
266 struct GNUNET_BIO_ReadHandle *ret;
268 fn = get_serialization_file_name (h, ext, ent);
271 ret = GNUNET_BIO_read_open (fn);
278 * Using the given serialization filename, try to deserialize
279 * the file-information tree associated with it.
281 * @param h master context
282 * @param filename name of the file (without directory) with
284 * @return NULL on error
286 static struct GNUNET_FS_FileInformation *
287 deserialize_file_information (struct GNUNET_FS_Handle *h,
288 const char *filename);
292 * Using the given serialization filename, try to deserialize
293 * the file-information tree associated with it.
295 * @param h master context
296 * @param fn name of the file (without directory) with
298 * @param rh handle for reading
299 * @return NULL on error
301 static struct GNUNET_FS_FileInformation *
302 deserialize_fi_node (struct GNUNET_FS_Handle *h,
304 struct GNUNET_BIO_ReadHandle *rh)
306 struct GNUNET_FS_FileInformation *ret;
307 struct GNUNET_FS_FileInformation *nxt;
315 GNUNET_BIO_read (rh, "termination flag", &b, sizeof(b)))
320 ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation));
325 GNUNET_BIO_read_meta_data (rh, "metadata", &ret->meta)) ||
327 GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32*1024)) ||
329 (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL))) ||
331 GNUNET_FS_uri_test_ksk (ret->keywords)) ||
333 GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) ||
336 (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) ||
338 GNUNET_FS_uri_test_chk (ret->chk_uri)) ) ) ||
340 GNUNET_BIO_read_int64 (rh, &ret->expirationTime.value)) ||
342 GNUNET_BIO_read_int64 (rh, &ret->start_time.value)) ||
344 GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16*1024)) ||
346 GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16*1024)) ||
348 GNUNET_BIO_read_int32 (rh, &ret->anonymity)) ||
350 GNUNET_BIO_read_int32 (rh, &ret->priority)) )
354 case 0: /* file-insert */
356 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size))
358 ret->is_directory = GNUNET_NO;
359 ret->data.file.do_index = GNUNET_NO;
360 ret->data.file.have_hash = GNUNET_NO;
361 ret->data.file.index_start_confirmed = GNUNET_NO;
362 /* FIXME: what's our approach for dealing with the
363 'reader' and 'reader_cls' fields? I guess the only
364 good way would be to dump "small" files into
365 'rh' and to not support serialization of "large"
368 case 1: /* file-index, no hash */
370 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size))
372 ret->is_directory = GNUNET_NO;
373 ret->data.file.do_index = GNUNET_YES;
374 ret->data.file.have_hash = GNUNET_NO;
375 ret->data.file.index_start_confirmed = GNUNET_NO;
376 /* FIXME: what's our approach for dealing with the
377 'reader' and 'reader_cls' fields?
378 (should be easy for indexing since we must have a file) */
380 case 2: /* file-index-with-hash */
382 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
384 GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, sizeof (GNUNET_HashCode))) )
386 ret->is_directory = GNUNET_NO;
387 ret->data.file.do_index = GNUNET_YES;
388 ret->data.file.have_hash = GNUNET_YES;
389 ret->data.file.index_start_confirmed = GNUNET_NO;
390 /* FIXME: what's our approach for dealing with the
391 'reader' and 'reader_cls' fields?
392 (should be easy for indexing since we must have a file) */
394 case 3: /* file-index-with-hash-confirmed */
396 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
398 GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, sizeof (GNUNET_HashCode))) )
400 ret->is_directory = GNUNET_NO;
401 ret->data.file.do_index = GNUNET_YES;
402 ret->data.file.have_hash = GNUNET_YES;
403 ret->data.file.index_start_confirmed = GNUNET_YES;
404 /* FIXME: what's our approach for dealing with the
405 'reader' and 'reader_cls' fields?
406 (should be easy for indexing since we must have a file) */
408 case 4: /* directory */
410 GNUNET_BIO_read_int32 (rh, &dsize)) ||
411 (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) ||
413 GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) ||
415 GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16*1024)) )
417 ret->data.dir.dir_size = (uint32_t) dsize;
418 ret->is_directory = GNUNET_YES;
419 if (filename != NULL)
421 ret->data.dir.entries = deserialize_file_information (h, filename);
422 GNUNET_free (filename);
424 nxt = ret->data.dir.entries;
436 /* FIXME: adjust ret->start_time! */
437 ret->serialization = GNUNET_strdup (fn);
439 GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16*1024))
441 if (filename != NULL)
443 ret->next = deserialize_file_information (h, filename);
444 GNUNET_free (filename);
449 GNUNET_free_non_null (ksks);
450 GNUNET_free_non_null (chks);
451 GNUNET_free_non_null (filename);
452 GNUNET_FS_file_information_destroy (ret, NULL, NULL);
459 * Using the given serialization filename, try to deserialize
460 * the file-information tree associated with it.
462 * @param h master context
463 * @param filename name of the file (without directory) with
465 * @return NULL on error
467 static struct GNUNET_FS_FileInformation *
468 deserialize_file_information (struct GNUNET_FS_Handle *h,
469 const char *filename)
471 struct GNUNET_FS_FileInformation *ret;
472 struct GNUNET_BIO_ReadHandle *rh;
475 rh = get_read_handle (h, "publish-fi", filename);
478 ret = deserialize_fi_node (h, filename, rh);
480 GNUNET_BIO_read_close (rh, &emsg))
482 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
483 _("Failed to resume publishing information `%s': %s\n"),
493 * Find the entry in the file information struct where the
494 * serialization filename matches the given name.
496 * @param pos file information to search
497 * @param srch filename to search for
498 * @return NULL if srch was not found in this subtree
500 static struct GNUNET_FS_FileInformation *
501 find_file_position (struct GNUNET_FS_FileInformation *pos,
504 struct GNUNET_FS_FileInformation *r;
508 if (0 == strcmp (srch,
511 if (pos->is_directory)
513 r = find_file_position (pos->data.dir.entries,
525 * Signal the FS's progress function that we are resuming
528 * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
529 * @param fi the entry in the publish-structure
530 * @param length length of the file or directory
531 * @param meta metadata for the file or directory (can be modified)
532 * @param uri pointer to the keywords that will be used for this entry (can be modified)
533 * @param anonymity pointer to selected anonymity level (can be modified)
534 * @param priority pointer to selected priority (can be modified)
535 * @param expirationTime pointer to selected expiration time (can be modified)
536 * @param client_info pointer to client context set upon creation (can be modified)
537 * @return GNUNET_OK to continue (always)
540 fip_signal_resume(void *cls,
541 struct GNUNET_FS_FileInformation *fi,
543 struct GNUNET_CONTAINER_MetaData *meta,
544 struct GNUNET_FS_Uri **uri,
547 struct GNUNET_TIME_Absolute *expirationTime,
550 struct GNUNET_FS_PublishContext *sc = cls;
551 struct GNUNET_FS_ProgressInfo pi;
553 pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME;
554 pi.value.publish.specifics.resume.message = sc->fi->emsg;
555 pi.value.publish.specifics.resume.chk_uri = sc->fi->chk_uri;
556 *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
562 * Function called with a filename of serialized publishing operation
565 * @param cls the 'struct GNUNET_FS_Handle*'
566 * @param filename complete filename (absolute path)
567 * @return GNUNET_OK (continue to iterate)
570 deserialize_publish_file (void *cls,
571 const char *filename)
573 struct GNUNET_FS_Handle *h = cls;
574 struct GNUNET_BIO_ReadHandle *rh;
575 struct GNUNET_FS_PublishContext *pc;
583 rh = GNUNET_BIO_read_open (filename);
586 if (0 != UNLINK (filename))
587 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
597 pc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
600 GNUNET_BIO_read_string (rh, "publish-nid", &pc->nid, 1024)) ||
602 GNUNET_BIO_read_string (rh, "publish-nuid", &pc->nuid, 1024)) ||
604 GNUNET_BIO_read_int32 (rh, &options)) ||
606 GNUNET_BIO_read_int32 (rh, &all_done)) ||
608 GNUNET_BIO_read_string (rh, "publish-firoot", &fi_root, 128)) ||
610 GNUNET_BIO_read_string (rh, "publish-fipos", &fi_pos, 128)) ||
612 GNUNET_BIO_read_string (rh, "publish-ns", &ns, 1024)) )
614 GNUNET_free_non_null (pc->nid);
615 GNUNET_free_non_null (pc->nuid);
616 GNUNET_free_non_null (fi_root);
617 GNUNET_free_non_null (ns);
621 pc->options = options;
622 pc->all_done = all_done;
623 pc->fi = deserialize_file_information (h, fi_root);
626 GNUNET_free_non_null (pc->nid);
627 GNUNET_free_non_null (pc->nuid);
628 GNUNET_free_non_null (fi_root);
629 GNUNET_free_non_null (ns);
635 pc->namespace = GNUNET_FS_namespace_create (h, ns);
636 if (pc->namespace == NULL)
638 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
639 _("Failed to recover namespace `%s', cannot resume publishing operation.\n"),
641 GNUNET_free_non_null (pc->nid);
642 GNUNET_free_non_null (pc->nuid);
643 GNUNET_free_non_null (fi_root);
644 GNUNET_free_non_null (ns);
651 pc->fi_pos = find_file_position (pc->fi,
653 GNUNET_free (fi_pos);
654 if (pc->fi_pos == NULL)
656 /* failed to find position for resuming, outch! Will start from root! */
658 if (pc->all_done != GNUNET_YES)
662 pc->serialization = GNUNET_strdup (filename);
663 /* generate RESUME event(s) */
664 GNUNET_FS_file_information_inspect (pc->fi,
668 /* re-start publishing (if needed)... */
669 if (pc->all_done != GNUNET_YES)
671 = GNUNET_SCHEDULER_add_with_priority (h->sched,
672 GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
673 &GNUNET_FS_publish_main_,
677 GNUNET_BIO_read_close (rh, &emsg))
679 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
680 _("Failed to resume publishing operation `%s': %s\n"),
690 * Deserialize information about pending publish operations.
692 * @param h master context
695 deserialize_publish (struct GNUNET_FS_Handle *h)
699 dn = get_serialization_file_name (h, "publish", "");
702 GNUNET_DISK_directory_scan (dn, &deserialize_publish_file, h);
708 * Setup a connection to the file-sharing service.
710 * @param sched scheduler to use
711 * @param cfg configuration to use
712 * @param client_name unique identifier for this client
713 * @param upcb function to call to notify about FS actions
714 * @param upcb_cls closure for upcb
715 * @param flags specific attributes for fs-operations
716 * @param ... list of optional options, terminated with GNUNET_FS_OPTIONS_END
717 * @return NULL on error
719 struct GNUNET_FS_Handle *
720 GNUNET_FS_start (struct GNUNET_SCHEDULER_Handle *sched,
721 const struct GNUNET_CONFIGURATION_Handle *cfg,
722 const char *client_name,
723 GNUNET_FS_ProgressCallback upcb,
725 enum GNUNET_FS_Flags flags,
728 struct GNUNET_FS_Handle *ret;
729 struct GNUNET_CLIENT_Connection *client;
730 enum GNUNET_FS_OPTIONS opt;
733 client = GNUNET_CLIENT_connect (sched,
738 ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Handle));
741 ret->client_name = GNUNET_strdup (client_name);
743 ret->upcb_cls = upcb_cls;
744 ret->client = client;
746 ret->max_parallel_downloads = 1;
747 ret->max_parallel_requests = 1;
748 ret->avg_block_latency = GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */
749 va_start (ap, flags);
750 while (GNUNET_FS_OPTIONS_END != (opt = va_arg (ap, enum GNUNET_FS_OPTIONS)))
754 case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM:
755 ret->max_parallel_downloads = va_arg (ap, unsigned int);
757 case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM:
758 ret->max_parallel_requests = va_arg (ap, unsigned int);
762 GNUNET_free (ret->client_name);
769 // FIXME: setup receive-loop with client (do we need one?)
771 if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags))
773 deserialize_publish (ret);
774 /* FIXME: not implemented! */
775 // Deserialize Search:
776 // * read search queries
777 // * for each query, read file with search results
778 // * for each search result with active download, deserialize download
779 // * for each directory search result, check for active downloads of contents
780 // Deserialize Download:
781 // * always part of search???
782 // Deserialize Unindex:
783 // * read FNs for unindex with progress offset
790 * Close our connection with the file-sharing service.
791 * The callback given to GNUNET_FS_start will no longer be
792 * called after this function returns.
794 * @param h handle that was returned from GNUNET_FS_start
797 GNUNET_FS_stop (struct GNUNET_FS_Handle *h)
799 if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & h->flags))
801 // FIXME: generate SUSPEND events and clean up state!
803 // FIXME: terminate receive-loop with client (do we need one?)
804 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
805 GNUNET_SCHEDULER_cancel (h->sched,
807 GNUNET_CLIENT_disconnect (h->client, GNUNET_NO);
808 GNUNET_free (h->client_name);