2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
23 * @brief main FS functions
24 * @author Christian Grothoff
28 #include "gnunet_util_lib.h"
29 #include "gnunet_fs_service.h"
34 * Start the given job (send signal, remove from pending queue, update
35 * counters and state).
37 * @param qe job to start
40 start_job (struct GNUNET_FS_QueueEntry *qe)
42 qe->client = GNUNET_CLIENT_connect (qe->h->sched, "fs", qe->h->cfg);
43 if (qe->client == NULL)
48 qe->start (qe->cls, qe->client);
50 qe->h->active_blocks += qe->blocks;
51 qe->start_time = GNUNET_TIME_absolute_get ();
52 GNUNET_CONTAINER_DLL_remove (qe->h->pending_head,
55 GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head,
63 * Stop the given job (send signal, remove from active queue, update
64 * counters and state).
66 * @param qe job to stop
69 stop_job (struct GNUNET_FS_QueueEntry *qe)
73 qe->h->active_downloads--;
74 qe->h->active_blocks -= qe->blocks;
75 qe->run_time = GNUNET_TIME_relative_add (qe->run_time,
76 GNUNET_TIME_absolute_get_duration (qe->start_time));
77 GNUNET_CONTAINER_DLL_remove (qe->h->running_head,
80 GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head,
88 * Process the jobs in the job queue, possibly starting some
89 * and stopping others.
91 * @param cls the 'struct GNUNET_FS_Handle'
92 * @param tc scheduler context
95 process_job_queue (void *cls,
96 const struct GNUNET_SCHEDULER_TaskContext *tc)
98 struct GNUNET_FS_Handle *h = cls;
99 struct GNUNET_FS_QueueEntry *qe;
100 struct GNUNET_FS_QueueEntry *next;
101 struct GNUNET_TIME_Relative run_time;
102 struct GNUNET_TIME_Relative restart_at;
103 struct GNUNET_TIME_Relative rst;
104 struct GNUNET_TIME_Absolute end_time;
106 h->queue_job = GNUNET_SCHEDULER_NO_TASK;
107 next = h->pending_head;
108 while (NULL != (qe = next))
111 if (h->running_head == NULL)
116 if ( (qe->blocks + h->active_blocks <= h->max_parallel_requests) &&
117 (h->active_downloads + 1 <= h->max_parallel_downloads) )
123 if (h->pending_head == NULL)
124 return; /* no need to stop anything */
125 restart_at = GNUNET_TIME_UNIT_FOREVER_REL;
126 next = h->running_head;
127 while (NULL != (qe = next))
130 /* FIXME: might be faster/simpler to do this calculation only once
131 when we start a job (OTOH, this would allow us to dynamically
132 and easily adjust qe->blocks over time, given the right API...) */
133 run_time = GNUNET_TIME_relative_multiply (h->avg_block_latency,
134 qe->blocks * qe->start_times);
135 end_time = GNUNET_TIME_absolute_add (qe->start_time,
137 rst = GNUNET_TIME_absolute_get_remaining (end_time);
138 restart_at = GNUNET_TIME_relative_min (rst, restart_at);
143 h->queue_job = GNUNET_SCHEDULER_add_delayed (h->sched,
151 * Add a job to the queue.
153 * @param h handle to the overall FS state
154 * @param start function to call to begin the job
155 * @param stop function to call to pause the job, or on dequeue (if the job was running)
156 * @param cls closure for start and stop
157 * @param blocks number of blocks this jobs uses
158 * @return queue handle
160 struct GNUNET_FS_QueueEntry *
161 GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h,
162 GNUNET_FS_QueueStart start,
163 GNUNET_FS_QueueStop stop,
167 struct GNUNET_FS_QueueEntry *qe;
169 qe = GNUNET_malloc (sizeof (struct GNUNET_FS_QueueEntry));
174 qe->queue_time = GNUNET_TIME_absolute_get ();
176 GNUNET_CONTAINER_DLL_insert_after (h->pending_head,
180 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
181 GNUNET_SCHEDULER_cancel (h->sched,
184 = GNUNET_SCHEDULER_add_now (h->sched,
192 * Dequeue a job from the queue.
193 * @param qh handle for the job
196 GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh)
198 struct GNUNET_FS_Handle *h;
201 if (qh->client != NULL)
203 GNUNET_CONTAINER_DLL_remove (h->pending_head,
207 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
208 GNUNET_SCHEDULER_cancel (h->sched,
211 = GNUNET_SCHEDULER_add_now (h->sched,
218 * Return the full filename where we would store state information
219 * (for serialization/deserialization).
221 * @param h master context
222 * @param ext component of the path
223 * @param ent entity identifier (or emtpy string for the directory)
224 * @return NULL on error
227 get_serialization_file_name (struct GNUNET_FS_Handle *h,
234 GNUNET_CONFIGURATION_get_value_filename (h->cfg,
239 GNUNET_asprintf (&ret,
247 GNUNET_free (basename);
253 * Return a read handle for deserialization.
255 * @param h master context
256 * @param ext component of the path
257 * @param ent entity identifier (or emtpy string for the directory)
258 * @return NULL on error
260 static struct GNUNET_BIO_ReadHandle *
261 get_read_handle (struct GNUNET_FS_Handle *h,
266 struct GNUNET_BIO_ReadHandle *ret;
268 fn = get_serialization_file_name (h, ext, ent);
271 ret = GNUNET_BIO_read_open (fn);
278 * Using the given serialization filename, try to deserialize
279 * the file-information tree associated with it.
281 * @param h master context
282 * @param filename name of the file (without directory) with
284 * @return NULL on error
286 static struct GNUNET_FS_FileInformation *
287 deserialize_file_information (struct GNUNET_FS_Handle *h,
288 const char *filename);
292 * Using the given serialization filename, try to deserialize
293 * the file-information tree associated with it.
295 * @param h master context
296 * @param fn name of the file (without directory) with
298 * @param rh handle for reading
299 * @return NULL on error
301 static struct GNUNET_FS_FileInformation *
302 deserialize_fi_node (struct GNUNET_FS_Handle *h,
304 struct GNUNET_BIO_ReadHandle *rh)
306 struct GNUNET_FS_FileInformation *ret;
307 struct GNUNET_FS_FileInformation *nxt;
315 GNUNET_BIO_read (rh, "termination flag", &b, sizeof(b)))
320 ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation));
325 GNUNET_BIO_read_meta_data (rh, "metadata", &ret->meta)) ||
327 GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32*1024)) ||
329 (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL))) ||
331 GNUNET_FS_uri_test_ksk (ret->keywords)) ||
333 GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) ||
336 (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) ||
338 GNUNET_FS_uri_test_chk (ret->chk_uri)) ) ) ||
340 GNUNET_BIO_read_int64 (rh, &ret->expirationTime.value)) ||
342 GNUNET_BIO_read_int64 (rh, &ret->start_time.value)) ||
344 GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16*1024)) ||
346 GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16*1024)) ||
348 GNUNET_BIO_read_int32 (rh, &ret->anonymity)) ||
350 GNUNET_BIO_read_int32 (rh, &ret->priority)) )
354 case 0: /* file-insert */
356 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size))
358 ret->is_directory = GNUNET_NO;
359 ret->data.file.do_index = GNUNET_NO;
360 ret->data.file.have_hash = GNUNET_NO;
361 ret->data.file.index_start_confirmed = GNUNET_NO;
363 case 1: /* file-index, no hash */
365 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size))
367 ret->is_directory = GNUNET_NO;
368 ret->data.file.do_index = GNUNET_YES;
369 ret->data.file.have_hash = GNUNET_NO;
370 ret->data.file.index_start_confirmed = GNUNET_NO;
372 case 2: /* file-index-with-hash */
374 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
376 GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, sizeof (GNUNET_HashCode))) )
378 ret->is_directory = GNUNET_NO;
379 ret->data.file.do_index = GNUNET_YES;
380 ret->data.file.have_hash = GNUNET_YES;
381 ret->data.file.index_start_confirmed = GNUNET_NO;
383 case 3: /* file-index-with-hash-confirmed */
385 GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) ||
387 GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, sizeof (GNUNET_HashCode))) )
389 ret->is_directory = GNUNET_NO;
390 ret->data.file.do_index = GNUNET_YES;
391 ret->data.file.have_hash = GNUNET_YES;
392 ret->data.file.index_start_confirmed = GNUNET_YES;
394 case 4: /* directory */
396 GNUNET_BIO_read_int32 (rh, &dsize)) ||
397 (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) ||
399 GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) ||
401 GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16*1024)) )
403 ret->data.dir.dir_size = (uint32_t) dsize;
404 ret->is_directory = GNUNET_YES;
405 if (filename != NULL)
407 ret->data.dir.entries = deserialize_file_information (h, filename);
408 GNUNET_free (filename);
410 nxt = ret->data.dir.entries;
422 /* FIXME: adjust ret->start_time! */
423 ret->serialization = GNUNET_strdup (fn);
425 GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16*1024))
427 if (filename != NULL)
429 ret->next = deserialize_file_information (h, filename);
430 GNUNET_free (filename);
435 GNUNET_free_non_null (ksks);
436 GNUNET_free_non_null (chks);
437 GNUNET_free_non_null (filename);
438 GNUNET_FS_file_information_destroy (ret, NULL, NULL);
445 * Using the given serialization filename, try to deserialize
446 * the file-information tree associated with it.
448 * @param h master context
449 * @param filename name of the file (without directory) with
451 * @return NULL on error
453 static struct GNUNET_FS_FileInformation *
454 deserialize_file_information (struct GNUNET_FS_Handle *h,
455 const char *filename)
457 struct GNUNET_FS_FileInformation *ret;
458 struct GNUNET_BIO_ReadHandle *rh;
461 rh = get_read_handle (h, "publish-fi", filename);
464 ret = deserialize_fi_node (h, filename, rh);
466 GNUNET_BIO_read_close (rh, &emsg))
468 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
469 _("Failed to resume publishing information `%s': %s\n"),
479 * Find the entry in the file information struct where the
480 * serialization filename matches the given name.
482 * @param pos file information to search
483 * @param srch filename to search for
484 * @return NULL if srch was not found in this subtree
486 static struct GNUNET_FS_FileInformation *
487 find_file_position (struct GNUNET_FS_FileInformation *pos,
490 struct GNUNET_FS_FileInformation *r;
494 if (0 == strcmp (srch,
497 if (pos->is_directory)
499 r = find_file_position (pos->data.dir.entries,
511 * Signal the FS's progress function that we are resuming
514 * @param cls closure (of type "struct GNUNET_FS_PublishContext*")
515 * @param fi the entry in the publish-structure
516 * @param length length of the file or directory
517 * @param meta metadata for the file or directory (can be modified)
518 * @param uri pointer to the keywords that will be used for this entry (can be modified)
519 * @param anonymity pointer to selected anonymity level (can be modified)
520 * @param priority pointer to selected priority (can be modified)
521 * @param expirationTime pointer to selected expiration time (can be modified)
522 * @param client_info pointer to client context set upon creation (can be modified)
523 * @return GNUNET_OK to continue (always)
526 fip_signal_resume(void *cls,
527 struct GNUNET_FS_FileInformation *fi,
529 struct GNUNET_CONTAINER_MetaData *meta,
530 struct GNUNET_FS_Uri **uri,
533 struct GNUNET_TIME_Absolute *expirationTime,
536 struct GNUNET_FS_PublishContext *sc = cls;
537 struct GNUNET_FS_ProgressInfo pi;
539 pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME;
540 pi.value.publish.specifics.resume.message = sc->fi->emsg;
541 pi.value.publish.specifics.resume.chk_uri = sc->fi->chk_uri;
542 *client_info = GNUNET_FS_publish_make_status_ (&pi, sc, fi, 0);
548 * Function called with a filename of serialized publishing operation
551 * @param cls the 'struct GNUNET_FS_Handle*'
552 * @param filename complete filename (absolute path)
553 * @return GNUNET_OK (continue to iterate)
556 deserialize_publish_file (void *cls,
557 const char *filename)
559 struct GNUNET_FS_Handle *h = cls;
560 struct GNUNET_BIO_ReadHandle *rh;
561 struct GNUNET_FS_PublishContext *pc;
569 rh = GNUNET_BIO_read_open (filename);
572 if (0 != UNLINK (filename))
573 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
583 pc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext));
586 GNUNET_BIO_read_string (rh, "publish-nid", &pc->nid, 1024)) ||
588 GNUNET_BIO_read_string (rh, "publish-nuid", &pc->nuid, 1024)) ||
590 GNUNET_BIO_read_int32 (rh, &options)) ||
592 GNUNET_BIO_read_int32 (rh, &all_done)) ||
594 GNUNET_BIO_read_string (rh, "publish-firoot", &fi_root, 128)) ||
596 GNUNET_BIO_read_string (rh, "publish-fipos", &fi_pos, 128)) ||
598 GNUNET_BIO_read_string (rh, "publish-ns", &ns, 1024)) )
600 GNUNET_free_non_null (pc->nid);
601 GNUNET_free_non_null (pc->nuid);
602 GNUNET_free_non_null (fi_root);
603 GNUNET_free_non_null (ns);
607 pc->options = options;
608 pc->all_done = all_done;
609 pc->fi = deserialize_file_information (h, fi_root);
612 GNUNET_free_non_null (pc->nid);
613 GNUNET_free_non_null (pc->nuid);
614 GNUNET_free_non_null (fi_root);
615 GNUNET_free_non_null (ns);
621 pc->namespace = GNUNET_FS_namespace_create (h, ns);
622 if (pc->namespace == NULL)
624 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
625 _("Failed to recover namespace `%s', cannot resume publishing operation.\n"),
627 GNUNET_free_non_null (pc->nid);
628 GNUNET_free_non_null (pc->nuid);
629 GNUNET_free_non_null (fi_root);
630 GNUNET_free_non_null (ns);
637 pc->fi_pos = find_file_position (pc->fi,
639 GNUNET_free (fi_pos);
640 if (pc->fi_pos == NULL)
642 /* failed to find position for resuming, outch! Will start from root! */
644 if (pc->all_done != GNUNET_YES)
648 pc->serialization = GNUNET_strdup (filename);
649 /* generate RESUME event(s) */
650 GNUNET_FS_file_information_inspect (pc->fi,
654 /* re-start publishing (if needed)... */
655 if (pc->all_done != GNUNET_YES)
657 = GNUNET_SCHEDULER_add_with_priority (h->sched,
658 GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
659 &GNUNET_FS_publish_main_,
663 GNUNET_BIO_read_close (rh, &emsg))
665 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
666 _("Failed to resume publishing operation `%s': %s\n"),
676 * Deserialize information about pending publish operations.
678 * @param h master context
681 deserialize_publish (struct GNUNET_FS_Handle *h)
685 dn = get_serialization_file_name (h, "publish", "");
688 GNUNET_DISK_directory_scan (dn, &deserialize_publish_file, h);
694 * Setup a connection to the file-sharing service.
696 * @param sched scheduler to use
697 * @param cfg configuration to use
698 * @param client_name unique identifier for this client
699 * @param upcb function to call to notify about FS actions
700 * @param upcb_cls closure for upcb
701 * @param flags specific attributes for fs-operations
702 * @param ... list of optional options, terminated with GNUNET_FS_OPTIONS_END
703 * @return NULL on error
705 struct GNUNET_FS_Handle *
706 GNUNET_FS_start (struct GNUNET_SCHEDULER_Handle *sched,
707 const struct GNUNET_CONFIGURATION_Handle *cfg,
708 const char *client_name,
709 GNUNET_FS_ProgressCallback upcb,
711 enum GNUNET_FS_Flags flags,
714 struct GNUNET_FS_Handle *ret;
715 struct GNUNET_CLIENT_Connection *client;
716 enum GNUNET_FS_OPTIONS opt;
719 client = GNUNET_CLIENT_connect (sched,
724 ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Handle));
727 ret->client_name = GNUNET_strdup (client_name);
729 ret->upcb_cls = upcb_cls;
730 ret->client = client;
732 ret->max_parallel_downloads = 1;
733 ret->max_parallel_requests = 1;
734 ret->avg_block_latency = GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */
735 va_start (ap, flags);
736 while (GNUNET_FS_OPTIONS_END != (opt = va_arg (ap, enum GNUNET_FS_OPTIONS)))
740 case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM:
741 ret->max_parallel_downloads = va_arg (ap, unsigned int);
743 case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM:
744 ret->max_parallel_requests = va_arg (ap, unsigned int);
748 GNUNET_free (ret->client_name);
755 // FIXME: setup receive-loop with client (do we need one?)
757 if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags))
759 deserialize_publish (ret);
760 /* FIXME: not implemented! */
761 // Deserialize Search:
762 // * read search queries
763 // * for each query, read file with search results
764 // * for each search result with active download, deserialize download
765 // * for each directory search result, check for active downloads of contents
766 // Deserialize Download:
767 // * always part of search???
768 // Deserialize Unindex:
769 // * read FNs for unindex with progress offset
776 * Close our connection with the file-sharing service.
777 * The callback given to GNUNET_FS_start will no longer be
778 * called after this function returns.
780 * @param h handle that was returned from GNUNET_FS_start
783 GNUNET_FS_stop (struct GNUNET_FS_Handle *h)
785 if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & h->flags))
787 // FIXME: generate SUSPEND events and clean up state!
789 // FIXME: terminate receive-loop with client (do we need one?)
790 if (h->queue_job != GNUNET_SCHEDULER_NO_TASK)
791 GNUNET_SCHEDULER_cancel (h->sched,
793 GNUNET_CLIENT_disconnect (h->client, GNUNET_NO);
794 GNUNET_free (h->client_name);