2 This file is part of GNUnet.
3 (C) 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file arm/gnunet-service-arm.c
23 * @brief the automated restart manager service
24 * @author Christian Grothoff
27 * - multiple start-stop requests with RC>1 can result
28 * in UP/DOWN signals based on "pending" that are inaccurate...
29 * => have list of clients waiting for a resolution instead of
30 * giving instant (but incorrect) replies
31 * - need to test auto-restart code on configuration changes;
32 * - should refine restart code to check if *relevant* parts of the
33 * configuration were changed (anything in the section for the service)
34 * - should have a way to specify dependencies between services and
35 * manage restarts of groups of services
38 #include "gnunet_client_lib.h"
39 #include "gnunet_getopt_lib.h"
40 #include "gnunet_os_lib.h"
41 #include "gnunet_protocols.h"
42 #include "gnunet_service_lib.h"
43 #include "gnunet_signal_lib.h"
48 * Check for configuration file changes every 5s.
50 #define MAINT_FREQUENCY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
53 * Threshold after which exponential backoff shouldn't increase (in ms); 30m
55 #define EXPONENTIAL_BACKOFF_THRESHOLD (1000 * 60 * 30)
59 * List of our services.
64 * List of our services.
69 * This is a linked list.
71 struct ServiceList *next;
74 * Name of the service.
79 * Name of the binary used.
84 * Name of the configuration file used.
89 * Client to notify upon kill completion (waitpid), NULL
90 * if we should simply restart the process.
92 struct GNUNET_SERVER_Client *killing_client;
95 * Process ID of the child.
100 * Last time the config of this service was
106 * Process exponential backoff time
108 struct GNUNET_TIME_Relative backoff;
111 * Absolute time at which the process is scheduled to restart in case of death
113 struct GNUNET_TIME_Absolute restartAt;
117 * Reference counter (counts how many times we've been
118 * asked to start the service). We only actually stop
119 * it once rc hits zero.
127 * List of running services.
129 static struct ServiceList *running;
134 static const struct GNUNET_CONFIGURATION_Handle *cfg;
139 static struct GNUNET_SCHEDULER_Handle *sched;
142 * Command to prepend to each actual command.
144 static char *prefix_command;
147 * Option to append to each actual command.
149 static char *final_option;
152 * ID of task called whenever we get a SIGCHILD.
154 static GNUNET_SCHEDULER_TaskIdentifier child_death_task;
157 * ID of task called whenever the timeout for restarting a child
160 static GNUNET_SCHEDULER_TaskIdentifier child_restart_task;
163 * Context for our SIGCHILD handler.
165 static struct GNUNET_SIGNAL_Context *shc_chld;
168 * Pipe used to communicate shutdown via signal.
170 static struct GNUNET_DISK_PipeHandle *sigpipe;
173 * Reading end of the signal pipe.
175 static const struct GNUNET_DISK_FileHandle *pr;
178 * Are we in shutdown mode?
180 static int in_shutdown;
184 * Handle to our server instance. Our server is a bit special in that
185 * its service is not immediately stopped once we get a shutdown
186 * request (since we need to continue service until all of our child
187 * processes are dead). This handle is used to shut down the server
188 * (and thus trigger process termination) once all child processes are
189 * also dead. A special option in the ARM configuration modifies the
190 * behaviour of the service implementation to not do the shutdown
193 static struct GNUNET_SERVER_Handle *server;
197 * If the configuration file changes, restart tasks that depended on that
200 * @param cls closure, NULL if we need to self-restart
204 config_change_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
206 struct ServiceList *pos;
212 /* FIXME: this test for config change is a bit too coarse grained */
213 if ( (0 == STAT (pos->config, &sbuf)) &&
214 (pos->mtime < sbuf.st_mtime) &&
217 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
218 _("Restarting service `%s' due to configuration file change.\n"));
219 if (0 != PLIBC_KILL (pos->pid, SIGTERM))
220 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
222 pos->backoff = GNUNET_TIME_UNIT_MILLISECONDS;
231 * Transmit a status result message.
233 * @param cls pointer to "unit16_t*" with message type
234 * @param size number of bytes available in buf
235 * @param buf where to copy the message, NULL on error
236 * @return number of bytes copied to buf
239 write_result (void *cls, size_t size, void *buf)
242 struct GNUNET_MessageHeader *msg;
246 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
247 _("Could not send status result to client\n"));
248 return 0; /* error, not much we can do */
251 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
252 "Sending status response %u to client\n",
253 (unsigned int) *res);
255 GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
257 msg->size = htons (sizeof (struct GNUNET_MessageHeader));
258 msg->type = htons (*res);
260 return sizeof (struct GNUNET_MessageHeader);
266 * Signal our client that we will start or stop the
269 * @param client who is being signalled
270 * @param name name of the service
271 * @param result message type to send
272 * @return NULL if it was not found
275 signal_result (struct GNUNET_SERVER_Client *client,
276 const char *name, uint16_t result)
282 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
284 ("Not sending status result to client: no client known\n"));
288 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
289 "Telling client that service `%s' is now %s\n",
291 result == GNUNET_MESSAGE_TYPE_ARM_IS_DOWN ? "down" : "up");
293 res = GNUNET_malloc (sizeof (uint16_t));
295 GNUNET_SERVER_notify_transmit_ready (client,
296 sizeof (struct GNUNET_MessageHeader),
297 GNUNET_TIME_UNIT_FOREVER_REL,
303 * Find the process with the given service
304 * name in the given list, remove it and return it.
306 * @param name which service entry to look up
307 * @return NULL if it was not found
309 static struct ServiceList *
310 find_name (const char *name)
312 struct ServiceList *pos;
313 struct ServiceList *prev;
319 if (0 == strcmp (pos->name, name))
324 prev->next = pos->next;
336 * Free an entry in the service list.
338 * @param pos entry to free
341 free_entry (struct ServiceList *pos)
343 GNUNET_free_non_null (pos->config);
344 GNUNET_free_non_null (pos->binary);
345 GNUNET_free (pos->name);
349 #include "do_start_process.c"
352 * Actually start the process for the given service.
354 * @param sl identifies service to start
357 start_process (struct ServiceList *sl)
370 GNUNET_CONFIGURATION_get_value_string (cfg,
371 sl->name, "PREFIX", &loprefix))
372 loprefix = GNUNET_strdup (prefix_command);
374 GNUNET_CONFIGURATION_get_value_string (cfg,
375 sl->name, "OPTIONS", &options))
377 options = GNUNET_strdup (final_option);
378 if (NULL == strstr (options, "%"))
380 /* replace '{}' with service name */
381 while (NULL != (optpos = strstr (options, "{}")))
385 GNUNET_asprintf (&optpos,
388 GNUNET_free (options);
391 /* replace '$PATH' with value associated with "PATH" */
392 while (NULL != (optpos = strstr (options, "$")))
395 while (isupper (*optend)) optend++;
403 GNUNET_CONFIGURATION_get_value_string (cfg, "PATHS",
406 val = GNUNET_strdup ("");
408 GNUNET_asprintf (&optpos,
414 GNUNET_free (options);
420 use_debug = GNUNET_CONFIGURATION_get_value_yesno (cfg, sl->name, "DEBUG");
422 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Starting service `%s'\n"), sl->name);
424 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
425 "Starting service `%s' using binary `%s' and configuration `%s'\n",
426 sl->name, sl->binary, sl->config);
428 if (GNUNET_YES == use_debug)
429 sl->pid = do_start_process (loprefix,
436 sl->pid = do_start_process (loprefix,
441 GNUNET_free (loprefix);
442 GNUNET_free (options);
443 /* FIXME: should check sl->pid */
448 * Start the specified service.
450 * @param client who is asking for this
451 * @param servicename name of the service to start
454 start_service (struct GNUNET_SERVER_Client *client, const char *servicename)
456 struct ServiceList *sl;
461 if (GNUNET_YES == in_shutdown)
463 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
464 _("ARM is shutting down, service `%s' not started.\n"),
466 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
469 sl = find_name (servicename);
472 /* already running, just increment RC */
473 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
474 _("Service `%s' already running.\n"), servicename);
480 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_UP);
484 GNUNET_CONFIGURATION_get_value_string (cfg,
485 servicename, "BINARY", &binary))
487 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
488 _("Binary implementing service `%s' not known!\n"),
490 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
494 GNUNET_CONFIGURATION_get_value_filename (cfg,
498 (0 != STAT (config, &sbuf)))
500 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
501 _("Configuration file `%s' for service `%s' not known!\n"),
502 config, servicename);
503 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
504 GNUNET_free (binary);
505 GNUNET_free_non_null (config);
508 sl = GNUNET_malloc (sizeof (struct ServiceList));
509 sl->name = GNUNET_strdup (servicename);
516 sl->mtime = sbuf.st_mtime;
517 sl->backoff = GNUNET_TIME_UNIT_MILLISECONDS;
518 sl->restartAt = GNUNET_TIME_UNIT_FOREVER_ABS;
523 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_UP);
528 * Stop the specified service.
530 * @param client who is asking for this
531 * @param servicename name of the service to stop
534 stop_service (struct GNUNET_SERVER_Client *client, const char *servicename)
536 struct ServiceList *pos;
538 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
539 _("Preparing to stop `%s'\n"), servicename);
540 pos = find_name (servicename);
543 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_UNKNOWN);
544 GNUNET_SERVER_receive_done (client, GNUNET_OK);
550 /* RC>1, just decrement RC */
555 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
556 "Service `%s' still used by %u clients, will keep it running!\n",
557 servicename, pos->rc);
559 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_UP);
560 GNUNET_SERVER_receive_done (client, GNUNET_OK);
564 pos->rc--; /* decrement RC to zero */
566 if (pos->killing_client != NULL)
568 /* killing already in progress */
570 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
571 "Service `%s' is already down\n", servicename);
573 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
574 GNUNET_SERVER_receive_done (client, GNUNET_OK);
580 if (GNUNET_YES == in_shutdown)
583 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
584 "Termination request already sent to `%s' (since ARM is in shutdown).\n",
587 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
588 GNUNET_SERVER_receive_done (client, GNUNET_OK);
595 /* process is in delayed restart, simply remove it! */
597 signal_result (client, servicename, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
598 GNUNET_SERVER_receive_done (client, GNUNET_OK);
602 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
603 "Sending kill signal to service `%s', waiting for process to die.\n",
606 if (0 != PLIBC_KILL (pos->pid, SIGTERM))
607 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
610 pos->killing_client = client;
611 GNUNET_SERVER_client_keep (client);
616 * Handle START-message.
618 * @param cls closure (always NULL)
619 * @param client identification of the client
620 * @param message the actual message
621 * @return GNUNET_OK to keep the connection open,
622 * GNUNET_SYSERR to close it (signal serious error)
625 handle_start (void *cls,
626 struct GNUNET_SERVER_Client *client,
627 const struct GNUNET_MessageHeader *message)
629 const char *servicename;
632 size = ntohs (message->size);
633 size -= sizeof (struct GNUNET_MessageHeader);
634 servicename = (const char *) &message[1];
635 if ((size == 0) || (servicename[size - 1] != '\0'))
638 GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
641 start_service (client, servicename);
642 GNUNET_SERVER_receive_done (client, GNUNET_OK);
647 * Handle STOP-message.
649 * @param cls closure (always NULL)
650 * @param client identification of the client
651 * @param message the actual message
652 * @return GNUNET_OK to keep the connection open,
653 * GNUNET_SYSERR to close it (signal serious error)
656 handle_stop (void *cls,
657 struct GNUNET_SERVER_Client *client,
658 const struct GNUNET_MessageHeader *message)
660 const char *servicename;
663 size = ntohs (message->size);
664 size -= sizeof (struct GNUNET_MessageHeader);
665 servicename = (const char *) &message[1];
666 if ((size == 0) || (servicename[size - 1] != '\0'))
669 GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
672 stop_service (client, servicename);
677 * Remove all entries for tasks that are not running
678 * (pid = 0) from the running list (they will no longer
679 * be restarted since we are shutting down).
684 struct ServiceList *pos;
685 struct ServiceList *next;
686 struct ServiceList *prev;
709 * We are done with everything. Stop remaining
710 * tasks, signal handler and the server.
715 GNUNET_SERVER_destroy (server);
717 GNUNET_SIGNAL_handler_uninstall (shc_chld);
719 GNUNET_SCHEDULER_cancel (sched, child_death_task);
720 child_death_task = GNUNET_SCHEDULER_NO_TASK;
725 * Task run for shutdown.
727 * @param cls closure, NULL if we need to self-restart
731 shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
733 struct ServiceList *pos;
736 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Stopping all services\n"));
738 in_shutdown = GNUNET_YES;
745 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
746 "Sending SIGTERM to `%s'\n", pos->name);
748 if (0 != PLIBC_KILL (pos->pid, SIGTERM))
749 GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
759 * Task run whenever it is time to restart a child that died.
761 * @param cls closure, always NULL
765 delayed_restart_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
767 struct ServiceList *pos;
768 struct GNUNET_TIME_Relative lowestRestartDelay;
770 child_restart_task = GNUNET_SCHEDULER_NO_TASK;
771 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
778 lowestRestartDelay = GNUNET_TIME_UNIT_FOREVER_REL;
780 /* check for services that need to be restarted due to
781 configuration changes or because the last restart failed */
785 if ( (pos->pid == 0) &&
786 (GNUNET_YES != in_shutdown) )
788 if (GNUNET_TIME_absolute_get_remaining (pos->restartAt).value == 0)
790 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
791 _("Restarting service `%s'.\n"), pos->name);
797 = GNUNET_TIME_relative_min (lowestRestartDelay,
798 GNUNET_TIME_absolute_get_remaining
804 if (lowestRestartDelay.value != GNUNET_TIME_UNIT_FOREVER_REL.value)
807 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
808 "Will restart process in %llums\n",
809 (unsigned long long) lowestRestartDelay.value);
812 = GNUNET_SCHEDULER_add_delayed (sched,
814 &delayed_restart_task,
821 * Task triggered whenever we receive a SIGCHLD (child
824 * @param cls closure, NULL if we need to self-restart
828 maint_child_death (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
830 struct ServiceList *pos;
831 struct ServiceList *prev;
832 struct ServiceList *next;
837 enum GNUNET_OS_ProcessStatusType statusType;
838 unsigned long statusCode;
840 child_death_task = GNUNET_SCHEDULER_NO_TASK;
841 if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
844 GNUNET_SCHEDULER_add_read_file (sched, GNUNET_TIME_UNIT_FOREVER_REL, pr,
845 &maint_child_death, NULL);
848 /* consume the signal */
849 GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof (c)));
851 /* check for services that died (WAITPID) */
854 while (NULL != (pos = next))
862 if ((GNUNET_SYSERR == (ret = GNUNET_OS_process_status (pos->pid,
865 ( (ret == GNUNET_NO) ||
866 (statusType == GNUNET_OS_PROCESS_STOPPED) ||
867 (statusType == GNUNET_OS_PROCESS_RUNNING)) )
873 if (statusType == GNUNET_OS_PROCESS_EXITED)
875 statstr = _( /* process termination method */ "exit");
876 statcode = statusCode;
878 else if (statusType == GNUNET_OS_PROCESS_SIGNALED)
880 statstr = _( /* process termination method */ "signal");
881 statcode = statusCode;
885 statstr = _( /* process termination method */ "unknown");
889 if (NULL != pos->killing_client)
895 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
896 _("Service `%s' stopped\n"),
898 signal_result (pos->killing_client,
899 pos->name, GNUNET_MESSAGE_TYPE_ARM_IS_DOWN);
900 GNUNET_SERVER_receive_done (pos->killing_client, GNUNET_OK);
901 GNUNET_SERVER_client_drop (pos->killing_client);
905 if (GNUNET_YES != in_shutdown)
907 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
909 ("Service `%s' terminated with status %s/%d, will try to restart it!\n"),
910 pos->name, statstr, statcode);
911 /* schedule restart */
913 = GNUNET_TIME_relative_to_absolute (pos->backoff);
914 if (pos->backoff.value < EXPONENTIAL_BACKOFF_THRESHOLD)
916 = GNUNET_TIME_relative_multiply (pos->backoff, 2);
917 if (GNUNET_SCHEDULER_NO_TASK != child_restart_task)
918 GNUNET_SCHEDULER_cancel (sched, child_restart_task);
920 = GNUNET_SCHEDULER_add_with_priority (sched,
921 GNUNET_SCHEDULER_PRIORITY_IDLE,
922 &delayed_restart_task,
927 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
928 "Service `%s' terminated with status %s/%d\n",
929 pos->name, statstr, statcode);
935 if ( (running == NULL) &&
938 GNUNET_SERVER_destroy (server);
939 GNUNET_SIGNAL_handler_uninstall (shc_chld);
945 GNUNET_SCHEDULER_add_read_file (sched, GNUNET_TIME_UNIT_FOREVER_REL, pr,
946 &maint_child_death, NULL);
952 * List of handlers for the messages understood by this service.
954 static struct GNUNET_SERVER_MessageHandler handlers[] = {
955 {&handle_start, NULL, GNUNET_MESSAGE_TYPE_ARM_START, 0},
956 {&handle_stop, NULL, GNUNET_MESSAGE_TYPE_ARM_STOP, 0},
961 * Signal handler called for SIGCHLD. Triggers the
962 * respective handler by writing to the trigger pipe.
965 sighandler_child_death ()
970 GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle
971 (sigpipe, GNUNET_DISK_PIPE_END_WRITE), &c,
977 * Process arm requests.
980 * @param s scheduler to use
981 * @param serv the initialized server
982 * @param c configuration to use
986 struct GNUNET_SCHEDULER_Handle *s,
987 struct GNUNET_SERVER_Handle *serv,
988 const struct GNUNET_CONFIGURATION_Handle *c)
990 char *defaultservices;
996 GNUNET_assert (serv != NULL);
997 shc_chld = GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
998 GNUNET_assert (sigpipe == NULL);
999 sigpipe = GNUNET_DISK_pipe (GNUNET_NO);
1000 GNUNET_assert (sigpipe != NULL);
1001 pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
1002 GNUNET_assert (pr != NULL);
1003 GNUNET_SERVER_ignore_shutdown (serv, GNUNET_YES);
1004 GNUNET_SCHEDULER_add_delayed (sched,
1005 GNUNET_TIME_UNIT_FOREVER_REL,
1009 GNUNET_SCHEDULER_add_read_file (sched, GNUNET_TIME_UNIT_FOREVER_REL, pr,
1010 &maint_child_death, NULL);
1013 GNUNET_CONFIGURATION_get_value_string (cfg,
1017 prefix_command = GNUNET_strdup ("");
1019 GNUNET_CONFIGURATION_get_value_string (cfg,
1023 final_option = GNUNET_strdup ("");
1024 /* start default services... */
1026 GNUNET_CONFIGURATION_get_value_string (cfg,
1032 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1033 "Starting default services `%s'\n", defaultservices);
1035 pos = strtok (defaultservices, " ");
1038 start_service (NULL, pos);
1039 pos = strtok (NULL, " ");
1041 GNUNET_free (defaultservices);
1046 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1047 "No default services configured.\n");
1051 /* process client requests */
1052 GNUNET_SERVER_add_handlers (server, handlers);
1054 /* manage services */
1055 GNUNET_SCHEDULER_add_with_priority (sched,
1056 GNUNET_SCHEDULER_PRIORITY_IDLE,
1057 &config_change_task, NULL);
1062 * The main function for the arm service.
1064 * @param argc number of arguments from the command line
1065 * @param argv command line arguments
1066 * @return 0 ok, 1 on error
1069 main (int argc, char *const *argv)
1071 return (GNUNET_OK ==
1072 GNUNET_SERVICE_run (argc,
1073 argv, "arm", GNUNET_YES, &run, NULL)) ? 0 : 1;
1076 /* end of gnunet-service-arm.c */