asserts
[oweals/gnunet.git] / src / util / server.c
index 339dfae7c62916d094056af960af8a07ca925afb..131c7c5c1fdc01daead809e0b12426b7ac2061b6 100644 (file)
@@ -34,6 +34,7 @@
 #include "gnunet_server_lib.h"
 #include "gnunet_time_lib.h"
 #include "gnunet_disk_lib.h"
+#include "gnunet_protocols.h"
 
 #define DEBUG_SERVER GNUNET_NO
 
@@ -111,6 +112,12 @@ struct GNUNET_SERVER_Handle
    */
   void *access_cls;
 
+  /**
+   * NULL-terminated array of sockets used to listen for new
+   * connections.
+   */
+  struct GNUNET_NETWORK_Handle **listen_sockets;
+
   /**
    * After how long should an idle connection time
    * out (on write).
@@ -123,27 +130,22 @@ struct GNUNET_SERVER_Handle
   size_t maxbuf;
 
   /**
-   * Pipe used to signal shutdown of the server.
+   * Task scheduled to do the listening.
    */
-  struct GNUNET_DISK_PipeHandle *shutpipe;
+  GNUNET_SCHEDULER_TaskIdentifier listen_task;
 
   /**
-   * Socket used to listen for new connections.  Set to
-   * "-1" by GNUNET_SERVER_destroy to initiate shutdown.
+   * Do we ignore messages of types that we do not understand or do we
+   * require that a handler is found (and if not kill the connection)?
    */
-  struct GNUNET_NETWORK_Handle *listen_socket;
-
-  /**
-   * Set to GNUNET_YES if we are shutting down.
-   */
-  int do_shutdown;
+  int require_found;
 
   /**
-   * Do we ignore messages of types that we do not
-   * understand or do we require that a handler
-   * is found (and if not kill the connection)?
+   * Should all of the clients of this server continue to process
+   * connections as usual even if we get a shutdown request? (the
+   * listen socket always ignores shutdown).
    */
-  int require_found;
+  int clients_ignore_shutdown;
 
 };
 
@@ -190,7 +192,7 @@ struct GNUNET_SERVER_Client
    */
   GNUNET_SERVER_TransmitReadyCallback notify_transmit_ready;
 
-   /**
+  /**
    * Callback to ask about transmit-ready notification.
    */
   GNUNET_SERVER_TransmitReadyCancelCallback notify_transmit_ready_cancel;
@@ -211,6 +213,11 @@ struct GNUNET_SERVER_Client
    */
   char *side_buf;
 
+  /**
+   * ID of task used to restart processing.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier restart_task;
+
   /**
    * Number of bytes in the side buffer.
    */
@@ -222,12 +229,6 @@ struct GNUNET_SERVER_Client
    */
   struct GNUNET_TIME_Absolute last_activity;
 
-  /**
-   * Current task identifier for the receive call
-   * (or GNUNET_SCHEDULER_NO_TASK for none).
-   */
-  GNUNET_SCHEDULER_TaskIdentifier my_receive;
-
   /**
    * How many bytes in the "incoming_buffer" are currently
    * valid? (starting at offset 0).
@@ -264,50 +265,26 @@ struct GNUNET_SERVER_Client
    */
   int shutdown_now;
 
-};
-
-
-/**
- * Server has been asked to shutdown, free resources.
- */
-static void
-destroy_server (struct GNUNET_SERVER_Handle *server)
-{
-  struct GNUNET_SERVER_Client *pos;
-  struct HandlerList *hpos;
-  struct NotifyList *npos;
+  /**
+   * Are we currently trying to receive?
+   */
+  int receive_pending;
 
-#if DEBUG_SERVER
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Server shutting down.\n");
-#endif
-  GNUNET_assert (server->listen_socket == NULL);
-  if (GNUNET_OK != GNUNET_DISK_pipe_close (server->shutpipe))
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
-                        "pipe-close");
-  while (server->clients != NULL)
-    {
-      pos = server->clients;
-      server->clients = pos->next;
-      pos->server = NULL;
-    }
-  while (NULL != (hpos = server->handlers))
-    {
-      server->handlers = hpos->next;
-      GNUNET_free (hpos);
-    }
-  while (NULL != (npos = server->disconnect_notify_list))
-    {
-      server->disconnect_notify_list = npos->next;
-      GNUNET_free (npos);
-    }
-  GNUNET_free (server);
-}
+  /**
+   * Persist the file handle for this client no matter what happens,
+   * force the OS to close once the process actually dies.  Should only
+   * be used in special cases!
+   */
+  int persist;
+};
 
 
 /**
- * Scheduler says our listen socket is ready.
- * Process it!
+ * Scheduler says our listen socket is ready.  Process it!
+ *
+ * @param cls handle to our server for which we are processing the listen
+ *        socket
+ * @param tc reason why we are running right now
  */
 static void
 process_listen_socket (void *cls,
@@ -317,48 +294,60 @@ process_listen_socket (void *cls,
   struct GNUNET_CONNECTION_Handle *sock;
   struct GNUNET_SERVER_Client *client;
   struct GNUNET_NETWORK_FDSet *r;
-  const struct GNUNET_DISK_FileHandle *shutpipe;
+  unsigned int i;
 
-  if ((server->do_shutdown) ||
-      ((tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0))
+  server->listen_task = GNUNET_SCHEDULER_NO_TASK;
+  r = GNUNET_NETWORK_fdset_create ();
+  i = 0;
+  while (NULL != server->listen_sockets[i])
+    GNUNET_NETWORK_fdset_set (r, server->listen_sockets[i++]);
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     {
-      /* shutdown was initiated */
-      GNUNET_assert (server->listen_socket != NULL);
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (server->listen_socket));
-      server->listen_socket = NULL;
-      if (server->do_shutdown)
-        destroy_server (server);
+      /* ignore shutdown, someone else will take care of it! */
+      server->listen_task = GNUNET_SCHEDULER_add_select (server->sched,
+                                                         GNUNET_SCHEDULER_PRIORITY_HIGH,
+                                                         GNUNET_SCHEDULER_NO_TASK,
+                                                         GNUNET_TIME_UNIT_FOREVER_REL,
+                                                         r, NULL,
+                                                         &process_listen_socket,
+                                                         server);
+      GNUNET_NETWORK_fdset_destroy (r);
       return;
     }
-  shutpipe = GNUNET_DISK_pipe_handle (server->shutpipe, 0);
-  GNUNET_assert (GNUNET_NETWORK_fdset_isset (tc->read_ready, server->listen_socket));
-  GNUNET_assert (!GNUNET_NETWORK_fdset_handle_isset (tc->read_ready, shutpipe));
-  sock = GNUNET_CONNECTION_create_from_accept (tc->sched,
-                                                   server->access,
-                                                   server->access_cls,
-                                                   server->listen_socket,
-                                                   server->maxbuf);
-  if (sock != NULL)
+  i = 0;
+  while (NULL != server->listen_sockets[i])
     {
+      if (GNUNET_NETWORK_fdset_isset
+          (tc->read_ready, server->listen_sockets[i]))
+        {
+          sock =
+            GNUNET_CONNECTION_create_from_accept (tc->sched, server->access,
+                                                  server->access_cls,
+                                                  server->listen_sockets[i],
+                                                  server->maxbuf);
+          if (sock != NULL)
+            {
 #if DEBUG_SERVER
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Server accepted incoming connection.\n");
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                          "Server accepted incoming connection.\n");
 #endif
-      client = GNUNET_SERVER_connect_socket (server, sock);
-      /* decrement reference count, we don't keep "client" alive */
-      GNUNET_SERVER_client_drop (client);
+              client = GNUNET_SERVER_connect_socket (server, sock);
+              GNUNET_CONNECTION_ignore_shutdown (sock,
+                                                 server->clients_ignore_shutdown);
+              /* decrement reference count, we don't keep "client" alive */
+              GNUNET_SERVER_client_drop (client);
+            }
+        }
+      i++;
     }
   /* listen for more! */
-  r = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_set (r, server->listen_socket);
-  GNUNET_NETWORK_fdset_handle_set (r, shutpipe);
-  GNUNET_SCHEDULER_add_select (server->sched,
-                               GNUNET_YES,
-                               GNUNET_SCHEDULER_PRIORITY_HIGH,
-                               GNUNET_SCHEDULER_NO_TASK,
-                               GNUNET_TIME_UNIT_FOREVER_REL,
-                               r, NULL,
-                               &process_listen_socket, server);
+  server->listen_task = GNUNET_SCHEDULER_add_select (server->sched,
+                                                     GNUNET_SCHEDULER_PRIORITY_HIGH,
+                                                     GNUNET_SCHEDULER_NO_TASK,
+                                                     GNUNET_TIME_UNIT_FOREVER_REL,
+                                                     r, NULL,
+                                                     &process_listen_socket,
+                                                     server);
   GNUNET_NETWORK_fdset_destroy (r);
 }
 
@@ -366,6 +355,8 @@ process_listen_socket (void *cls,
 /**
  * Create and initialize a listen socket for the server.
  *
+ * @param serverAddr address to listen on
+ * @param socklen length of address
  * @return NULL on error, otherwise the listen socket
  */
 static struct GNUNET_NETWORK_Handle *
@@ -374,6 +365,7 @@ open_listen_socket (const struct sockaddr *serverAddr, socklen_t socklen)
   const static int on = 1;
   struct GNUNET_NETWORK_Handle *sock;
   uint16_t port;
+  int eno;
 
   switch (serverAddr->sa_family)
     {
@@ -384,44 +376,64 @@ open_listen_socket (const struct sockaddr *serverAddr, socklen_t socklen)
       port = ntohs (((const struct sockaddr_in6 *) serverAddr)->sin6_port);
       break;
     default:
-      GNUNET_break (0);
-      return NULL;
+      port = 0;
+      break;
     }
-  sock = GNUNET_NETWORK_socket_socket (serverAddr->sa_family, SOCK_STREAM, 0);
+  sock = GNUNET_NETWORK_socket_create (serverAddr->sa_family, SOCK_STREAM, 0);
   if (NULL == sock)
     {
       GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "socket");
+      errno = 0;
       return NULL;
     }
-#ifndef MINGW
-  if (GNUNET_OK != GNUNET_NETWORK_socket_set_inheritable (sock))
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                         "fcntl");
+  if (port != 0) 
+    {
+      if (GNUNET_NETWORK_socket_setsockopt
+         (sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
+       GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                            "setsockopt");
+#ifdef IPV6_V6ONLY
+      if ( (serverAddr->sa_family == AF_INET6) &&
+          (GNUNET_NETWORK_socket_setsockopt
+           (sock, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof (on)) != GNUNET_OK) )
+       GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                            "setsockopt");
 #endif
-  if (GNUNET_NETWORK_socket_setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                         "setsockopt");
+    }
   /* bind the socket */
   if (GNUNET_NETWORK_socket_bind (sock, serverAddr, socklen) != GNUNET_OK)
     {
-      GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "bind");
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _
-                  ("`%s' failed for port %d. Is the service already running?\n"),
-                  "bind", port);
+      eno = errno;
+      if (errno != EADDRINUSE)
+        {
+          /* we don't log 'EADDRINUSE' here since an IPv4 bind may
+             fail if we already took the port on IPv6; if both IPv4 and
+             IPv6 binds fail, then our caller will log using the
+             errno preserved in 'eno' */
+          GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "bind");
+          if (port != 0)
+            GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                        _
+                        ("`%s' failed for port %d (%s).\n"),
+                        "bind", port,
+                        (serverAddr->sa_family == AF_INET) ? "IPv4" : "IPv6");
+          eno = 0;
+        }
       GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (sock));
+      errno = eno;
       return NULL;
     }
   if (GNUNET_OK != GNUNET_NETWORK_socket_listen (sock, 5))
     {
       GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "listen");
       GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (sock));
+      errno = 0;
       return NULL;
     }
 #if DEBUG_SERVER
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Server starts to listen on port %u.\n",
-                 port);
+  if (port != 0)
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Server starts to listen on port %u.\n", port);
 #endif
   return sock;
 }
@@ -433,8 +445,7 @@ open_listen_socket (const struct sockaddr *serverAddr, socklen_t socklen)
  * @param sched scheduler to use
  * @param access function for access control
  * @param access_cls closure for access
- * @param serverAddr address to listen on (including port), use NULL
- *        for internal server (no listening)
+ * @param serverAddr address to listen on (including port), NULL terminated array
  * @param socklen length of serverAddr
  * @param maxbuf maximum write buffer size for accepted sockets
  * @param idle_timeout after how long should we timeout idle connections?
@@ -447,51 +458,67 @@ struct GNUNET_SERVER_Handle *
 GNUNET_SERVER_create (struct GNUNET_SCHEDULER_Handle *sched,
                       GNUNET_CONNECTION_AccessCheck access,
                       void *access_cls,
-                      const struct sockaddr *serverAddr,
-                      socklen_t socklen,
+                      struct sockaddr *const *serverAddr,
+                      const socklen_t * socklen,
                       size_t maxbuf,
                       struct GNUNET_TIME_Relative
                       idle_timeout, int require_found)
 {
   struct GNUNET_SERVER_Handle *ret;
-  struct GNUNET_NETWORK_Handle *lsock;
+  struct GNUNET_NETWORK_Handle **lsocks;
   struct GNUNET_NETWORK_FDSet *r;
+  unsigned int i;
+  unsigned int j;
 
-  lsock = NULL;
-  if (serverAddr != NULL)
+  i = 0;
+  while (serverAddr[i] != NULL)
+    i++;
+  if (i > 0)
     {
-      lsock = open_listen_socket (serverAddr, socklen);
-      if (lsock == NULL)
-        return NULL;
+      lsocks =
+        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (i + 1));
+      i = 0;
+      j = 0;
+      while (serverAddr[i] != NULL)
+        {
+          lsocks[j] = open_listen_socket (serverAddr[i], socklen[i]);
+          if (lsocks[j] != NULL)
+            j++;
+          i++;
+        }
+      if (j == 0)
+        {
+          if (errno != 0)
+           GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "bind");          
+          GNUNET_free (lsocks);
+          lsocks = NULL;
+        }
     }
-  ret = GNUNET_malloc (sizeof (struct GNUNET_SERVER_Handle));
-  ret->shutpipe = GNUNET_malloc (sizeof (struct GNUNET_DISK_FileDescriptor *[2]));
-  if (NULL == (ret->shutpipe = GNUNET_DISK_pipe (GNUNET_NO)))
+  else
     {
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (lsock));
-      GNUNET_free (ret->shutpipe);
-      GNUNET_free (ret);
-      return NULL;
+      lsocks = NULL;
     }
+  ret = GNUNET_malloc (sizeof (struct GNUNET_SERVER_Handle));
   ret->sched = sched;
   ret->maxbuf = maxbuf;
   ret->idle_timeout = idle_timeout;
-  ret->listen_socket = lsock;
+  ret->listen_sockets = lsocks;
   ret->access = access;
   ret->access_cls = access_cls;
   ret->require_found = require_found;
-  if (lsock != NULL)
+  if (lsocks != NULL)
     {
       r = GNUNET_NETWORK_fdset_create ();
-      GNUNET_NETWORK_fdset_set (r, ret->listen_socket);
-      GNUNET_NETWORK_fdset_handle_set (r, GNUNET_DISK_pipe_handle (ret->shutpipe, 0));
-      GNUNET_SCHEDULER_add_select (sched,
-                                   GNUNET_YES,
-                                   GNUNET_SCHEDULER_PRIORITY_HIGH,
-                                   GNUNET_SCHEDULER_NO_TASK,
-                                   GNUNET_TIME_UNIT_FOREVER_REL,
-                                   r,
-                                   NULL, &process_listen_socket, ret);
+      i = 0;
+      while (NULL != ret->listen_sockets[i])
+        GNUNET_NETWORK_fdset_set (r, ret->listen_sockets[i++]);
+      ret->listen_task = GNUNET_SCHEDULER_add_select (sched,
+                                                      GNUNET_SCHEDULER_PRIORITY_HIGH,
+                                                      GNUNET_SCHEDULER_NO_TASK,
+                                                      GNUNET_TIME_UNIT_FOREVER_REL,
+                                                      r, NULL,
+                                                      &process_listen_socket,
+                                                      ret);
       GNUNET_NETWORK_fdset_destroy (r);
     }
   return ret;
@@ -500,18 +527,47 @@ GNUNET_SERVER_create (struct GNUNET_SCHEDULER_Handle *sched,
 
 /**
  * Free resources held by this server.
+ *
+ * @param s server to destroy
  */
 void
 GNUNET_SERVER_destroy (struct GNUNET_SERVER_Handle *s)
 {
-  static char c;
+  struct HandlerList *hpos;
+  struct NotifyList *npos;
+  unsigned int i;
 
-  GNUNET_assert (s->do_shutdown == GNUNET_NO);
-  s->do_shutdown = GNUNET_YES;
-  if (s->listen_socket == NULL)
-    destroy_server (s);
-  else
-    GNUNET_break (1 == GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (s->shutpipe, 1), &c, 1));
+#if DEBUG_SERVER
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Server shutting down.\n");
+#endif
+  if (GNUNET_SCHEDULER_NO_TASK != s->listen_task)
+    {
+      GNUNET_SCHEDULER_cancel (s->sched, s->listen_task);
+      s->listen_task = GNUNET_SCHEDULER_NO_TASK;
+    }
+  if (s->listen_sockets != NULL)
+    {
+      i = 0;
+      while (s->listen_sockets[i] != NULL)
+        GNUNET_break (GNUNET_OK ==
+                      GNUNET_NETWORK_socket_close (s->listen_sockets[i++]));
+      GNUNET_free (s->listen_sockets);
+      s->listen_sockets = NULL;
+    }
+  while (s->clients != NULL)
+    GNUNET_SERVER_client_disconnect (s->clients);
+  while (NULL != (hpos = s->handlers))
+    {
+      s->handlers = hpos->next;
+      GNUNET_free (hpos);
+    }
+  while (NULL != (npos = s->disconnect_notify_list))
+    {
+      npos->callback (npos->callback_cls, NULL);
+      s->disconnect_notify_list = npos->next;
+      GNUNET_free (npos);
+    }
+  GNUNET_free (s);
 }
 
 
@@ -573,9 +629,8 @@ GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
   size = ntohs (message->size);
 #if DEBUG_SERVER
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Server schedules transmission of %u-byte message of type %u to client.\n",
-             size,
-             type);
+              "Server schedules transmission of %u-byte message of type %u to client.\n",
+              size, type);
 #endif
   pos = server->handlers;
   found = GNUNET_NO;
@@ -585,7 +640,8 @@ GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
       while (pos->handlers[i].callback != NULL)
         {
           mh = &pos->handlers[i];
-          if (mh->type == type)
+          if ( (mh->type == type) ||
+               (mh->type == GNUNET_MESSAGE_TYPE_ALL) )
             {
               if ((mh->expected_size != 0) && (mh->expected_size != size))
                 {
@@ -612,53 +668,6 @@ GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
 }
 
 
-/**
- * We're finished with this client and especially its input
- * processing.  If the RC is zero, free all resources otherwise wait
- * until RC hits zero to do so.
- */
-static void
-shutdown_incoming_processing (struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_SERVER_Client *prev;
-  struct GNUNET_SERVER_Client *pos;
-  struct GNUNET_SERVER_Handle *server;
-  struct NotifyList *n;
-  unsigned int rc;
-
-  GNUNET_assert (client->my_receive == GNUNET_SCHEDULER_NO_TASK);
-  rc = client->reference_count;
-  if (client->server != NULL)
-    {
-      server = client->server;
-      client->server = NULL;
-      prev = NULL;
-      pos = server->clients;
-      while ((pos != NULL) && (pos != client))
-        {
-          prev = pos;
-          pos = pos->next;
-        }
-      GNUNET_assert (pos != NULL);
-      if (prev == NULL)
-        server->clients = pos->next;
-      else
-        prev->next = pos->next;
-      n = server->disconnect_notify_list;
-      while (n != NULL)
-        {
-          n->callback (n->callback_cls, client);
-          n = n->next;
-        }
-    }
-  /* wait for RC to hit zero, then free */
-  if (rc > 0)
-    return;
-  client->destroy (client->client_closure);
-  GNUNET_free (client);
-}
-
-
 /**
  * Go over the contents of the client buffer; as long as full messages
  * are available, pass them on for processing.  Update the buffer
@@ -677,36 +686,35 @@ process_client_buffer (struct GNUNET_SERVER_Client *client)
   server = client->server;
 #if DEBUG_SERVER
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Private buffer contains %u bytes; client is %s and we are %s\n",
-             client->receive_pos,
-             client->suspended ? "suspended" : "up",
-             client->shutdown_now ? "in shutdown" : "running");
+              "Private buffer contains %u bytes; client is %s and we are %s\n",
+              client->receive_pos,
+              client->suspended ? "suspended" : "up",
+              client->shutdown_now ? "in shutdown" : "running");
 #endif
-  while ((client->receive_pos >= sizeof (struct GNUNET_MessageHeader)) &&
-         (0 == client->suspended) && (GNUNET_YES != client->shutdown_now))
+  while ( (client->receive_pos >= sizeof (struct GNUNET_MessageHeader)) &&
+         (0 == client->suspended) && 
+         (GNUNET_YES != client->shutdown_now) )
     {
       hdr = (const struct GNUNET_MessageHeader *) &client->incoming_buffer;
       msize = ntohs (hdr->size);
       if (msize > client->receive_pos)
-       {
+        {
 #if DEBUG_SERVER
-         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                     "Total message size is %u, we only have %u bytes; need more data\n",
-                     msize,
-                     client->receive_pos);
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      "Total message size is %u, we only have %u bytes; need more data\n",
+                      msize, client->receive_pos);
 #endif
-         break;
-       }
+          break;
+        }
 #if DEBUG_SERVER
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Passing %u bytes to callback for processing\n",
-                 msize);
+                  "Passing %u bytes to callback for processing\n", msize);
 #endif
-      if ((msize < sizeof (struct GNUNET_MessageHeader)) ||
-          (GNUNET_OK != GNUNET_SERVER_inject (server, client, hdr)))
+      if ( (msize < sizeof (struct GNUNET_MessageHeader)) ||
+          (GNUNET_OK != GNUNET_SERVER_inject (server, client, hdr)) )
         {
           client->in_process_client_buffer = GNUNET_NO;
-          shutdown_incoming_processing (client);
+          GNUNET_SERVER_client_disconnect (client);
           return;
         }
       /* FIXME: this is highly inefficient; we should
@@ -718,7 +726,7 @@ process_client_buffer (struct GNUNET_SERVER_Client *client)
     }
   client->in_process_client_buffer = GNUNET_NO;
   if (GNUNET_YES == client->shutdown_now)
-    shutdown_incoming_processing (client);
+    GNUNET_SERVER_client_disconnect (client);
 }
 
 
@@ -736,16 +744,14 @@ static void
 process_incoming (void *cls,
                   const void *buf,
                   size_t available,
-                  const struct sockaddr *addr, 
-                 socklen_t addrlen,
-                 int errCode)
+                  const struct sockaddr *addr, socklen_t addrlen, int errCode)
 {
   struct GNUNET_SERVER_Client *client = cls;
   struct GNUNET_SERVER_Handle *server = client->server;
   const char *cbuf = buf;
   size_t maxcpy;
 
-  client->my_receive = GNUNET_SCHEDULER_NO_TASK;
+  client->receive_pending = GNUNET_NO;
   if ((buf == NULL) ||
       (available == 0) ||
       (errCode != 0) ||
@@ -753,15 +759,14 @@ process_incoming (void *cls,
       (client->shutdown_now == GNUNET_YES) ||
       (GNUNET_YES != client->check (client->client_closure)))
     {
-      /* other side closed connection, error connecting, etc. */
-      shutdown_incoming_processing (client);
+      /* other side closed connection, error connecting, etc. */      
+      GNUNET_SERVER_client_disconnect (client);
       return;
     }
 #if DEBUG_SERVER
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Server receives %u bytes from `%s'.\n",
-             available,
-             GNUNET_a2s(addr, addrlen));
+              "Server receives %u bytes from `%s'.\n",
+              available, GNUNET_a2s (addr, addrlen));
 #endif
   GNUNET_SERVER_client_keep (client);
   client->last_activity = GNUNET_TIME_absolute_get ();
@@ -773,8 +778,7 @@ process_incoming (void *cls,
         maxcpy = sizeof (client->incoming_buffer) - client->receive_pos;
 #if DEBUG_SERVER
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Can copy %u bytes to private buffer\n",
-                 maxcpy);
+                  "Can copy %u bytes to private buffer\n", maxcpy);
 #endif
       memcpy (&client->incoming_buffer[client->receive_pos], cbuf, maxcpy);
       client->receive_pos += maxcpy;
@@ -785,12 +789,12 @@ process_incoming (void *cls,
           if (available > 0)
             {
 #if DEBUG_SERVER
-             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                         "Client has suspended processing; copying %u bytes to side buffer to be used later.\n",
-                         available);
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                          "Client has suspended processing; copying %u bytes to side buffer to be used later.\n",
+                          available);
 #endif
-             GNUNET_assert (client->side_buf_size == 0);
-             GNUNET_assert (client->side_buf == NULL);
+              GNUNET_assert (client->side_buf_size == 0);
+              GNUNET_assert (client->side_buf == NULL);
               client->side_buf_size = available;
               client->side_buf = GNUNET_malloc (available);
               memcpy (client->side_buf, cbuf, available);
@@ -800,7 +804,7 @@ process_incoming (void *cls,
         }
 #if DEBUG_SERVER
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Now processing messages in private buffer\n");
+                  "Now processing messages in private buffer\n");
 #endif
       process_client_buffer (client);
     }
@@ -809,31 +813,47 @@ process_incoming (void *cls,
       (GNUNET_YES != client->shutdown_now) && (client->server != NULL))
     {
       /* Finally, keep receiving! */
-      client->my_receive = client->receive (client->client_closure,
-                                            GNUNET_SERVER_MAX_MESSAGE_SIZE,
-                                            server->idle_timeout,
-                                            &process_incoming, client);
+      client->receive_pending = GNUNET_YES;
+      client->receive (client->client_closure,
+                       GNUNET_SERVER_MAX_MESSAGE_SIZE,
+                       server->idle_timeout, &process_incoming, client);
     }
   if (GNUNET_YES == client->shutdown_now)
-    shutdown_incoming_processing (client);
+    GNUNET_SERVER_client_disconnect (client);
   GNUNET_SERVER_client_drop (client);
 }
 
 
 /**
- * FIXME: document.
+ * Task run to start again receiving from the network
+ * and process requests.
+ *
+ * @param cls our 'struct GNUNET_SERVER_Client*' to process more requests from
+ * @param tc scheduler context (unused)
  */
 static void
 restart_processing (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_SERVER_Client *client = cls;
+  struct GNUNET_SERVER_Handle *server = client->server;
 
+  client->restart_task = GNUNET_SCHEDULER_NO_TASK;
+  if ( (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) &&
+       (GNUNET_NO == server->clients_ignore_shutdown) )
+    {
+      GNUNET_SERVER_client_disconnect (client);
+      return;
+    }
+  GNUNET_SERVER_client_keep (client);
   process_client_buffer (client);
   if (0 == client->suspended)
-    client->my_receive = client->receive (client->client_closure,
-                                          GNUNET_SERVER_MAX_MESSAGE_SIZE,
-                                          client->server->idle_timeout,
-                                          &process_incoming, client);
+    {
+      client->receive_pending = GNUNET_YES;
+      client->receive (client->client_closure,
+                      GNUNET_SERVER_MAX_MESSAGE_SIZE,
+                      client->server->idle_timeout, &process_incoming, client);
+    }
+  GNUNET_SERVER_client_drop (client);
 }
 
 
@@ -849,10 +869,10 @@ add_client (struct GNUNET_SERVER_Handle *server,
   client->last_activity = GNUNET_TIME_absolute_get ();
   client->next = server->clients;
   server->clients = client;
-  client->my_receive = client->receive (client->client_closure,
-                                        GNUNET_SERVER_MAX_MESSAGE_SIZE,
-                                        server->idle_timeout,
-                                        &process_incoming, client);
+  client->receive_pending = GNUNET_YES;
+  client->receive (client->client_closure,
+                   GNUNET_SERVER_MAX_MESSAGE_SIZE,
+                   server->idle_timeout, &process_incoming, client);
 }
 
 
@@ -864,15 +884,14 @@ add_client (struct GNUNET_SERVER_Handle *server,
  * @param timeout when should this operation time out
  * @param receiver function to call for processing
  * @param receiver_cls closure for receiver
- * @return task identifier that can be used to cancel the operation
  */
-static GNUNET_SCHEDULER_TaskIdentifier
+static void
 sock_receive (void *cls,
               size_t max,
               struct GNUNET_TIME_Relative timeout,
               GNUNET_CONNECTION_Receiver receiver, void *receiver_cls)
 {
-  return GNUNET_CONNECTION_receive (cls, max, timeout, receiver, receiver_cls);
+  GNUNET_CONNECTION_receive (cls, max, timeout, receiver, receiver_cls);
 }
 
 
@@ -880,12 +899,11 @@ sock_receive (void *cls,
  * Wrapper to cancel receiving from a socket.
  * 
  * @param cls handle to the GNUNET_CONNECTION_Handle to cancel
- * @param tc task ID that was returned by GNUNET_CONNECTION_receive
  */
 static void
-sock_receive_cancel (void *cls, GNUNET_SCHEDULER_TaskIdentifier ti)
+sock_receive_cancel (void *cls)
 {
-  GNUNET_CONNECTION_receive_cancel (cls, ti);
+  GNUNET_CONNECTION_receive_cancel (cls);
 }
 
 
@@ -900,7 +918,7 @@ sock_notify_transmit_ready (void *cls,
                             void *notify_cls)
 {
   return GNUNET_CONNECTION_notify_transmit_ready (cls, size, timeout, notify,
-                                               notify_cls);
+                                                  notify_cls);
 }
 
 
@@ -931,11 +949,16 @@ sock_check (void *cls)
  * Destroy this socket (free resources).
  *
  * @param cls the socket
+ * @param persist set the socket to be persisted
  */
 static void
-sock_destroy (void *cls)
+sock_destroy (void *cls, int persist)
 {
-  GNUNET_CONNECTION_destroy (cls);
+  struct GNUNET_CONNECTION_Handle *sock = cls;
+  if (persist == GNUNET_YES)
+    GNUNET_CONNECTION_persist_ (sock);
+
+  GNUNET_CONNECTION_destroy (sock, GNUNET_NO);
 }
 
 
@@ -1050,8 +1073,9 @@ GNUNET_SERVER_client_drop (struct GNUNET_SERVER_Client *client)
 {
   GNUNET_assert (client->reference_count > 0);
   client->reference_count--;
-  if ((client->server == NULL) && (client->reference_count == 0))
-    shutdown_incoming_processing (client);
+  if ( (client->shutdown_now == GNUNET_YES) && 
+       (client->reference_count == 0) )
+    GNUNET_SERVER_client_disconnect (client);
 }
 
 
@@ -1070,7 +1094,7 @@ GNUNET_SERVER_client_get_address (struct GNUNET_SERVER_Client *client,
   if (client->receive != &sock_receive)
     return GNUNET_SYSERR;       /* not a network client */
   return GNUNET_CONNECTION_get_address (client->client_closure,
-                                            addr, addrlen);
+                                        addr, addrlen);
 }
 
 
@@ -1099,6 +1123,44 @@ GNUNET_SERVER_disconnect_notify (struct GNUNET_SERVER_Handle *server,
 }
 
 
+/**
+ * Ask the server to stop notifying us whenever a client disconnects.
+ *
+ * @param server the server manageing the clients
+ * @param callback function to call on disconnect
+ * @param callback_cls closure for callback
+ */
+void
+GNUNET_SERVER_disconnect_notify_cancel (struct GNUNET_SERVER_Handle *server,
+                                       GNUNET_SERVER_DisconnectCallback callback,
+                                       void *callback_cls)
+{
+  struct NotifyList *pos;
+  struct NotifyList *prev;
+
+  prev = NULL;
+  pos = server->disconnect_notify_list;
+  while (pos != NULL)
+    {
+      if ( (pos->callback == callback) &&
+          (pos->callback_cls == callback_cls ) )
+       break;
+      prev = pos;
+      pos = pos->next;
+    }
+  if (pos == NULL)
+    {
+      GNUNET_break (0);
+      return;
+    }
+  if (prev == NULL)
+    server->disconnect_notify_list = pos->next;
+  else
+    prev->next = pos->next;
+  GNUNET_free (pos);
+}
+
+
 /**
  * Ask the server to disconnect from the given client.
  * This is the same as returning GNUNET_SYSERR from a message
@@ -1110,12 +1172,58 @@ GNUNET_SERVER_disconnect_notify (struct GNUNET_SERVER_Handle *server,
 void
 GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client)
 {
-  if (client->server == NULL)
-    return;                     /* already disconnected */
-  GNUNET_assert (client->my_receive != GNUNET_SCHEDULER_NO_TASK);
-  client->receive_cancel (client->client_closure, client->my_receive);
-  client->my_receive = GNUNET_SCHEDULER_NO_TASK;
-  shutdown_incoming_processing (client);
+  struct GNUNET_SERVER_Client *prev;
+  struct GNUNET_SERVER_Client *pos;
+  struct GNUNET_SERVER_Handle *server;
+  struct NotifyList *n;
+  unsigned int rc;
+
+  if (client->restart_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (client->server->sched,
+                              client->restart_task);
+      client->restart_task = GNUNET_SCHEDULER_NO_TASK;
+    }
+  if (GNUNET_YES == client->receive_pending)
+    {
+      client->receive_cancel (client->client_closure);
+      client->receive_pending = GNUNET_NO;
+    }
+
+  rc = client->reference_count;  
+  if (client->server != NULL)
+    {
+      server = client->server;
+      client->server = NULL;
+      client->shutdown_now = GNUNET_YES;
+      prev = NULL;
+      pos = server->clients;
+      while ((pos != NULL) && (pos != client))
+        {
+          prev = pos;
+          pos = pos->next;
+        }
+      GNUNET_assert (pos != NULL);
+      if (prev == NULL)
+        server->clients = pos->next;
+      else
+        prev->next = pos->next;
+      if (client->restart_task != GNUNET_SCHEDULER_NO_TASK)
+       GNUNET_SCHEDULER_cancel (server->sched,
+                                client->restart_task);
+      n = server->disconnect_notify_list;
+      while (n != NULL)
+        {
+          n->callback (n->callback_cls, client);
+          n = n->next;
+        }
+    }
+  if (rc > 0)
+    return;
+  if (client->in_process_client_buffer == GNUNET_YES)
+    return;
+  client->destroy (client->client_closure, client->persist);
+  GNUNET_free (client);  
 }
 
 
@@ -1123,7 +1231,6 @@ GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client)
  * Notify us when the server has enough space to transmit
  * a message of the given size to the given client.
  *
- * @param server the server to use
  * @param client client to transmit message to
  * @param size requested amount of buffer space
  * @param timeout after how long should we give up (and call
@@ -1147,6 +1254,17 @@ GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
                                         timeout, callback, callback_cls);
 }
 
+/**
+ * Set the persistent flag on this client, used to setup client connection
+ * to only be killed when the service it's connected to is actually dead.
+ *
+ * @param client the client to set the persistent flag on
+ */
+void
+GNUNET_SERVER_client_persist_ (struct GNUNET_SERVER_Client *client)
+{
+  client->persist = GNUNET_YES;
+}
 
 /**
  * Resume receiving from this client, we are done processing the
@@ -1156,6 +1274,7 @@ GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
  * @param client client we were processing a message of
  * @param success GNUNET_OK to keep the connection open and
  *                          continue to receive
+ *                GNUNET_NO to close the connection (normal behavior)
  *                GNUNET_SYSERR to close the connection (signal
  *                          serious error)
  */
@@ -1169,7 +1288,10 @@ GNUNET_SERVER_receive_done (struct GNUNET_SERVER_Client *client, int success)
   GNUNET_assert (client->suspended > 0);
   client->suspended--;
   if (success != GNUNET_OK)
-    client->shutdown_now = GNUNET_YES;
+    {
+      GNUNET_SERVER_client_disconnect (client);
+      return;
+    }
   if (client->suspended > 0)
     return;
   if (client->in_process_client_buffer == GNUNET_YES)
@@ -1180,26 +1302,37 @@ GNUNET_SERVER_receive_done (struct GNUNET_SERVER_Client *client, int success)
       sb = client->side_buf;
       client->side_buf = NULL;
       /* this will also resume the receive job */
-      if (GNUNET_YES != client->shutdown_now)
-        process_incoming (client, sb, client->side_buf_size, NULL, 0, 0);
-      else
-        shutdown_incoming_processing (client);
+      process_incoming (client, sb, client->side_buf_size, NULL, 0, 0);
       /* finally, free the side-buf */
       GNUNET_free (sb);
       return;
     }
-  /* resume receive job */
-  if (GNUNET_YES != client->shutdown_now)
+  if (client->server == NULL)
     {
-      GNUNET_SCHEDULER_add_continuation (client->server->sched,
-                                         GNUNET_NO,
-                                         &restart_processing,
-                                         client,
-                                         GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+      GNUNET_SERVER_client_disconnect (client);
       return;
     }
-  shutdown_incoming_processing (client);
+  client->restart_task = GNUNET_SCHEDULER_add_now (client->server->sched,
+                                                  &restart_processing,
+                                                  client);
 }
 
 
+/**
+ * Configure this server's connections to continue handling client
+ * requests as usual even after we get a shutdown signal.  The change
+ * only applies to clients that connect to the server from the outside
+ * using TCP after this call.  Clients managed previously or those
+ * added using GNUNET_SERVER_connect_socket and
+ * GNUNET_SERVER_connect_callback are not affected by this option.
+ *
+ * @param h server handle
+ * @param do_ignore GNUNET_YES to ignore, GNUNET_NO to restore default
+ */
+void
+GNUNET_SERVER_ignore_shutdown (struct GNUNET_SERVER_Handle *h, int do_ignore)
+{
+  h->clients_ignore_shutdown = do_ignore;
+}
+
 /* end of server.c */