Returns now GNUNET_SYSERR
[oweals/gnunet.git] / src / util / scheduler.c
index ae0428a94083efe3d09a384a04ae749fb31c13de..49d491bd7843546e48974b39f0a53159256dd52a 100644 (file)
  */
 #include "platform.h"
 #include "gnunet_common.h"
+#include "gnunet_os_lib.h"
 #include "gnunet_scheduler_lib.h"
 #include "gnunet_signal_lib.h"
 #include "gnunet_time_lib.h"
+#include "disk.h"
+#ifdef LINUX
+#include "execinfo.h"
+
+
+/**
+ * Use lsof to generate file descriptor reports on select error?
+ * (turn off for stable releases).
+ */
+#define USE_LSOF GNUNET_YES
+
+/**
+ * Obtain trace information for all scheduler calls that schedule tasks.
+ */
+#define EXECINFO GNUNET_NO
+
+/**
+ * Depth of the traces collected via EXECINFO.
+ */
+#define MAX_TRACE_DEPTH 50
+#endif
 
 #define DEBUG_TASKS GNUNET_NO
 
+/**
+ * Should we figure out which tasks are delayed for a while
+ * before they are run? (Consider using in combination with EXECINFO).
+ */
+#define PROFILE_DELAYS GNUNET_NO
+
+/**
+ * Task that were in the queue for longer than this are reported if
+ * PROFILE_DELAYS is active.
+ */
+#define DELAY_THRESHOLD GNUNET_TIME_UNIT_SECONDS
+
 /**
  * Linked list of pending tasks.
  */
@@ -60,10 +94,9 @@ struct Task
   struct GNUNET_NETWORK_FDSet *read_set;
 
   /**
-   * Set of file descriptors this task is waiting
-   * for for writing.  Once ready, this is updated
-   * to reflect the set of file descriptors ready
-   * for operation.
+   * Set of file descriptors this task is waiting for for writing.
+   * Once ready, this is updated to reflect the set of file
+   * descriptors ready for operation.
    */
   struct GNUNET_NETWORK_FDSet *write_set;
 
@@ -83,6 +116,13 @@ struct Task
    */
   struct GNUNET_TIME_Absolute timeout;
 
+#if PROFILE_DELAYS
+  /**
+   * When was the task scheduled?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+#endif
+
   /**
    * Why is the task ready?  Set after task is added to ready queue.
    * Initially set to zero.  All reasons that have already been
@@ -96,67 +136,98 @@ struct Task
   enum GNUNET_SCHEDULER_Priority priority;
 
   /**
-   * Should this task be run on shutdown?
+   * Set if we only wait for reading from a single FD, otherwise -1.
    */
-  int run_on_shutdown;
-
-};
-
-
-/**
- * Handle for the scheduling service.
- */
-struct GNUNET_SCHEDULER_Handle
-{
+  int read_fd;
 
   /**
-   * List of tasks waiting for an event.
+   * Set if we only wait for writing to a single FD, otherwise -1.
    */
-  struct Task *pending;
+  int write_fd;
 
+#if EXECINFO
   /**
-   * List of tasks ready to run right now,
-   * grouped by importance.
+   * Array of strings which make up a backtrace from the point when this
+   * task was scheduled (essentially, who scheduled the task?)
    */
-  struct Task *ready[GNUNET_SCHEDULER_PRIORITY_COUNT];
+  char **backtrace_strings;
 
   /**
-   * Identity of the last task queued.  Incremented for each task to
-   * generate a unique task ID (it is virtually impossible to start
-   * more than 2^64 tasks during the lifetime of a process).
+   * Size of the backtrace_strings array
    */
-  GNUNET_SCHEDULER_TaskIdentifier last_id;
+  int num_backtrace_strings;
+#endif
 
-  /**
-   * Highest number so that all tasks with smaller identifiers
-   * have already completed.  Also the lowest number of a task
-   * still waiting to be executed.
-   */
-  GNUNET_SCHEDULER_TaskIdentifier lowest_pending_id;
 
-  /**
-   * GNUNET_NO if we are running normally,
-   * GNUNET_YES if we are in shutdown mode.
-   */
-  int shutdown;
+};
 
-  /**
-   * Number of tasks on the ready list.
-   */
-  unsigned int ready_count;
 
-  /**
-   * How many tasks have we run so far?
  */
-  unsigned long long tasks_run;
+/**
+ * List of tasks waiting for an event.
+ */
+static struct Task *pending;
 
-  /**
-   * Priority of the task running right now.  Only
-   * valid while a task is running.
-   */
-  enum GNUNET_SCHEDULER_Priority current_priority;
+/**
+ * List of tasks waiting ONLY for a timeout event.
+ * Sorted by timeout (earliest first).  Used so that
+ * we do not traverse the list of these tasks when
+ * building select sets (we just look at the head
+ * to determine the respective timeout ONCE).
+ */
+static struct Task *pending_timeout;
 
-};
+/**
+ * Last inserted task waiting ONLY for a timeout event.
+ * Used to (heuristically) speed up insertion.
+ */
+static struct Task *pending_timeout_last;
+
+/**
+ * ID of the task that is running right now.
+ */
+static struct Task *active_task;
+
+/**
+ * List of tasks ready to run right now,
+ * grouped by importance.
+ */
+static struct Task *ready[GNUNET_SCHEDULER_PRIORITY_COUNT];
+
+/**
+ * Identity of the last task queued.  Incremented for each task to
+ * generate a unique task ID (it is virtually impossible to start
+ * more than 2^64 tasks during the lifetime of a process).
+ */
+static GNUNET_SCHEDULER_TaskIdentifier last_id;
+
+/**
+ * Highest number so that all tasks with smaller identifiers
+ * have already completed.  Also the lowest number of a task
+ * still waiting to be executed.
+ */
+static GNUNET_SCHEDULER_TaskIdentifier lowest_pending_id;
+
+/**
+ * Number of tasks on the ready list.
+ */
+static unsigned int ready_count;
+
+/**
+ * How many tasks have we run so far?
+ */
+static unsigned long long tasks_run;
+
+/**
+ * Priority of the task running right now.  Only
+ * valid while a task is running.
+ */
+static enum GNUNET_SCHEDULER_Priority current_priority;
+
+/**
+ * Priority of the highest task added in the current select
+ * iteration.
+ */
+static enum GNUNET_SCHEDULER_Priority max_priority_added;
 
 
 /**
@@ -181,22 +252,29 @@ check_priority (enum GNUNET_SCHEDULER_Priority p)
  * future), but only if the return value is "GNUNET_NO" (and
  * the "lowest_pending_id" check failed).
  *
- * @param sched the scheduler
  * @param id which task are we checking for
  * @return GNUNET_YES if so, GNUNET_NO if not
  */
 static int
-is_pending (struct GNUNET_SCHEDULER_Handle *sched,
-            GNUNET_SCHEDULER_TaskIdentifier id)
+is_pending (GNUNET_SCHEDULER_TaskIdentifier id)
 {
   struct Task *pos;
   enum GNUNET_SCHEDULER_Priority p;
   GNUNET_SCHEDULER_TaskIdentifier min;
 
-  if (id < sched->lowest_pending_id)
+  if (id < lowest_pending_id)
     return GNUNET_NO;
   min = -1;                     /* maximum value */
-  pos = sched->pending;
+  pos = pending;
+  while (pos != NULL)
+    {
+      if (pos->id == id)
+        return GNUNET_YES;
+      if (pos->id < min)
+        min = pos->id;
+      pos = pos->next;
+    }
+  pos = pending_timeout;
   while (pos != NULL)
     {
       if (pos->id == id)
@@ -207,7 +285,7 @@ is_pending (struct GNUNET_SCHEDULER_Handle *sched,
     }
   for (p = 0; p < GNUNET_SCHEDULER_PRIORITY_COUNT; p++)
     {
-      pos = sched->ready[p];
+      pos = ready[p];
       while (pos != NULL)
         {
           if (pos->id == id)
@@ -217,7 +295,7 @@ is_pending (struct GNUNET_SCHEDULER_Handle *sched,
           pos = pos->next;
         }
     }
-  sched->lowest_pending_id = min;
+  lowest_pending_id = min;
   return GNUNET_NO;
 }
 
@@ -225,41 +303,54 @@ is_pending (struct GNUNET_SCHEDULER_Handle *sched,
 /**
  * Update all sets and timeout for select.
  *
- * @param sched the scheduler
  * @param rs read-set, set to all FDs we would like to read (updated)
  * @param ws write-set, set to all FDs we would like to write (updated)
  * @param timeout next timeout (updated)
  */
 static void
-update_sets (struct GNUNET_SCHEDULER_Handle *sched,
-             struct GNUNET_NETWORK_FDSet * rs, 
-            struct GNUNET_NETWORK_FDSet * ws, 
-            struct GNUNET_TIME_Relative *timeout)
+update_sets (struct GNUNET_NETWORK_FDSet *rs,
+             struct GNUNET_NETWORK_FDSet *ws,
+             struct GNUNET_TIME_Relative *timeout)
 {
   struct Task *pos;
+  struct GNUNET_TIME_Absolute now;
+  struct GNUNET_TIME_Relative to;
 
-  pos = sched->pending;
+  now = GNUNET_TIME_absolute_get ();
+  pos = pending_timeout;
+  if (pos != NULL) 
+    {
+      to = GNUNET_TIME_absolute_get_difference (now, pos->timeout);
+      if (timeout->rel_value > to.rel_value)
+       *timeout = to;
+      if (pos->reason != 0)
+        *timeout = GNUNET_TIME_UNIT_ZERO;
+    }
+  pos = pending;
   while (pos != NULL)
     {
       if ((pos->prereq_id != GNUNET_SCHEDULER_NO_TASK) &&
-          (GNUNET_YES == is_pending (sched, pos->prereq_id)))
+          (GNUNET_YES == is_pending (pos->prereq_id)))
         {
           pos = pos->next;
           continue;
         }
-
-      if (pos->timeout.value != GNUNET_TIME_UNIT_FOREVER_ABS.value)
+      if (pos->timeout.abs_value != GNUNET_TIME_UNIT_FOREVER_ABS.abs_value)
         {
-          struct GNUNET_TIME_Relative to;
-
-          to = GNUNET_TIME_absolute_get_remaining (pos->timeout);
-          if (timeout->value > to.value)
-              *timeout = to;
+          to = GNUNET_TIME_absolute_get_difference (now, pos->timeout);
+          if (timeout->rel_value > to.rel_value)
+            *timeout = to;
         }
+      if (pos->read_fd != -1)
+       GNUNET_NETWORK_fdset_set_native (rs, pos->read_fd);
+      if (pos->write_fd != -1)
+       GNUNET_NETWORK_fdset_set_native (ws, pos->write_fd);
       if (pos->read_set != NULL)
-       GNUNET_NETWORK_fdset_add (rs, pos->read_set);
+        GNUNET_NETWORK_fdset_add (rs, pos->read_set);
       if (pos->write_set != NULL)
-       GNUNET_NETWORK_fdset_add (ws, pos->write_set);
+        GNUNET_NETWORK_fdset_add (ws, pos->write_set);
+      if (pos->reason != 0)
+        *timeout = GNUNET_TIME_UNIT_ZERO;
       pos = pos->next;
     }
 }
@@ -275,10 +366,10 @@ update_sets (struct GNUNET_SCHEDULER_Handle *sched,
  * @return GNUNET_YES if there was some overlap
  */
 static int
-set_overlaps (const struct GNUNET_NETWORK_FDSet * ready, 
-             struct GNUNET_NETWORK_FDSet * want)
+set_overlaps (const struct GNUNET_NETWORK_FDSet *ready,
+              struct GNUNET_NETWORK_FDSet *want)
 {
-  if (NULL == want)
+  if ( (NULL == want) || (NULL == ready) )
     return GNUNET_NO;
   if (GNUNET_NETWORK_fdset_overlap (ready, want))
     {
@@ -295,7 +386,6 @@ set_overlaps (const struct GNUNET_NETWORK_FDSet * ready,
  * Check if the given task is eligible to run now.
  * Also set the reason why it is eligible.
  *
- * @param sched the scheduler
  * @param task task to check if it is ready
  * @param now the current time
  * @param rs set of FDs ready for reading
@@ -303,33 +393,38 @@ set_overlaps (const struct GNUNET_NETWORK_FDSet * ready,
  * @return GNUNET_YES if we can run it, GNUNET_NO if not.
  */
 static int
-is_ready (struct GNUNET_SCHEDULER_Handle *sched,
-          struct Task *task,
+is_ready (struct Task *task,
           struct GNUNET_TIME_Absolute now,
           const struct GNUNET_NETWORK_FDSet *rs,
-         const struct GNUNET_NETWORK_FDSet * ws)
+          const struct GNUNET_NETWORK_FDSet *ws)
 {
-  if ((GNUNET_NO == task->run_on_shutdown) && (GNUNET_YES == sched->shutdown))
-    return GNUNET_NO;
-  if ((GNUNET_YES == task->run_on_shutdown) &&
-      (GNUNET_YES == sched->shutdown))
-    task->reason |= GNUNET_SCHEDULER_REASON_SHUTDOWN;
-  if (now.value >= task->timeout.value)
-    task->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
-  if ((0 == (task->reason & GNUNET_SCHEDULER_REASON_READ_READY)) &&
-      (rs != NULL) && (set_overlaps (rs, task->read_set)))
-    task->reason |= GNUNET_SCHEDULER_REASON_READ_READY;
-  if ((0 == (task->reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) &&
-      (ws != NULL) && (set_overlaps (ws, task->write_set)))
-    task->reason |= GNUNET_SCHEDULER_REASON_WRITE_READY;
-  if (task->reason == 0)
-    return GNUNET_NO;           /* not ready */
+  enum GNUNET_SCHEDULER_Reason reason;
+
+  reason = task->reason;
+  if (now.abs_value >= task->timeout.abs_value)
+    reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
+  if ( (0 == (reason & GNUNET_SCHEDULER_REASON_READ_READY)) &&
+       ( ( (task->read_fd != -1) &&
+          (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (rs, task->read_fd)) ) ||
+        (set_overlaps (rs, task->read_set) ) ) )
+    reason |= GNUNET_SCHEDULER_REASON_READ_READY;
+  if ((0 == (reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) &&
+      ( ( (task->write_fd != -1) &&
+         (GNUNET_YES == GNUNET_NETWORK_fdset_test_native (ws, task->write_fd)) ) ||
+       (set_overlaps (ws, task->write_set) ) ) )
+    reason |= GNUNET_SCHEDULER_REASON_WRITE_READY;
+  if (reason == 0)
+    return GNUNET_NO;           /* not ready */    
   if (task->prereq_id != GNUNET_SCHEDULER_NO_TASK)
     {
-      if (GNUNET_YES == is_pending (sched, task->prereq_id))
-        return GNUNET_NO;       /* prereq waiting */
-      task->reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE;
+      if (GNUNET_YES == is_pending (task->prereq_id))
+       {
+         task->reason = reason;
+         return GNUNET_NO;       /* prereq waiting */
+       }
+      reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE;
     }
+  task->reason = reason;
   return GNUNET_YES;
 }
 
@@ -341,12 +436,14 @@ is_ready (struct GNUNET_SCHEDULER_Handle *sched,
  * @param task task ready for execution
  */
 static void
-queue_ready_task (struct GNUNET_SCHEDULER_Handle *handle, 
-                 struct Task *task)
+queue_ready_task (struct Task *task)
 {
-  task->next = handle->ready[check_priority (task->priority)];
-  handle->ready[check_priority (task->priority)] = task;
-  handle->ready_count++;
+  enum GNUNET_SCHEDULER_Priority p = task->priority;
+  if (0 != (task->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    p = GNUNET_SCHEDULER_PRIORITY_SHUTDOWN;
+  task->next = ready[check_priority (p)];
+  ready[check_priority (p)] = task;
+  ready_count++;
 }
 
 
@@ -359,9 +456,8 @@ queue_ready_task (struct GNUNET_SCHEDULER_Handle *handle,
  * @param ws FDs ready for writing
  */
 static void
-check_ready (struct GNUNET_SCHEDULER_Handle *handle,
-             const struct GNUNET_NETWORK_FDSet * rs,
-            const struct GNUNET_NETWORK_FDSet * ws)
+check_ready (const struct GNUNET_NETWORK_FDSet *rs,
+             const struct GNUNET_NETWORK_FDSet *ws)
 {
   struct Task *pos;
   struct Task *prev;
@@ -370,23 +466,36 @@ check_ready (struct GNUNET_SCHEDULER_Handle *handle,
 
   now = GNUNET_TIME_absolute_get ();
   prev = NULL;
-  pos = handle->pending;
+  pos = pending_timeout;
+  while (pos != NULL)
+    {
+      next = pos->next;
+      if (now.abs_value >= pos->timeout.abs_value)
+       pos->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
+      if (0 == pos->reason)
+       break;
+      pending_timeout = next;
+      if (pending_timeout_last == pos)
+       pending_timeout_last = NULL;
+      queue_ready_task (pos);
+      pos = next;
+    }
+  pos = pending;
   while (pos != NULL)
     {
 #if DEBUG_TASKS
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Checking readyness of task: %llu / %p\n",
-                 pos->id,
-                 pos->callback_cls);
+                  "Checking readiness of task: %llu / %p\n",
+                  pos->id, pos->callback_cls);
 #endif
       next = pos->next;
-      if (GNUNET_YES == is_ready (handle, pos, now, rs, ws))
+      if (GNUNET_YES == is_ready (pos, now, rs, ws))
         {
           if (prev == NULL)
-            handle->pending = next;
+            pending = next;
           else
             prev->next = next;
-          queue_ready_task (handle, pos);
+          queue_ready_task (pos);
           pos = next;
           continue;
         }
@@ -396,17 +505,67 @@ check_ready (struct GNUNET_SCHEDULER_Handle *handle,
 }
 
 
+/**
+ * Request the shutdown of a scheduler.  Marks all currently
+ * pending tasks as ready because of shutdown.  This will
+ * cause all tasks to run (as soon as possible, respecting
+ * priorities and prerequisite tasks).  Note that tasks
+ * scheduled AFTER this call may still be delayed arbitrarily.
+ */
+void
+GNUNET_SCHEDULER_shutdown ()
+{
+  struct Task *pos;
+  int i;
+
+  pos = pending_timeout;
+  while (pos != NULL)
+    {
+      pos->reason |= GNUNET_SCHEDULER_REASON_SHUTDOWN;
+      /* we don't move the task into the ready queue yet; check_ready
+         will do that later, possibly adding additional
+         readiness-factors */
+      pos = pos->next;
+    }
+  pos = pending;
+  while (pos != NULL)
+    {
+      pos->reason |= GNUNET_SCHEDULER_REASON_SHUTDOWN;
+      /* we don't move the task into the ready queue yet; check_ready
+         will do that later, possibly adding additional
+         readiness-factors */
+      pos = pos->next;
+    }
+  for (i=0;i<GNUNET_SCHEDULER_PRIORITY_COUNT;i++)
+    {
+      pos = ready[i];
+      while (pos != NULL)
+       {
+         pos->reason |= GNUNET_SCHEDULER_REASON_SHUTDOWN;
+         /* we don't move the task into the ready queue yet; check_ready
+            will do that later, possibly adding additional
+            readiness-factors */
+         pos = pos->next;
+       }
+    }  
+}
+
+
 /**
  * Destroy a task (release associated resources)
  *
  * @param t task to destroy
  */
-static void destroy_task (struct Task *t)
+static void
+destroy_task (struct Task *t)
 {
   if (NULL != t->read_set)
     GNUNET_NETWORK_fdset_destroy (t->read_set);
   if (NULL != t->write_set)
     GNUNET_NETWORK_fdset_destroy (t->write_set);
+#if EXECINFO
+  GNUNET_free (t->backtrace_strings);
+#endif
   GNUNET_free (t);
 }
 
@@ -416,59 +575,104 @@ static void destroy_task (struct Task *t)
  * empty.  Keep running tasks until we are either no longer running
  * "URGENT" tasks or until we have at least one "pending" task (which
  * may become ready, hence we should select on it).  Naturally, if
- * there are no more ready tasks, we also return.
+ * there are no more ready tasks, we also return.  
  *
- * @param sched the scheduler
+ * @param rs FDs ready for reading
+ * @param ws FDs ready for writing
  */
 static void
-run_ready (struct GNUNET_SCHEDULER_Handle *sched)
+run_ready (struct GNUNET_NETWORK_FDSet *rs,
+          struct GNUNET_NETWORK_FDSet *ws)
 {
   enum GNUNET_SCHEDULER_Priority p;
   struct Task *pos;
   struct GNUNET_SCHEDULER_TaskContext tc;
 
+  max_priority_added = GNUNET_SCHEDULER_PRIORITY_KEEP;
   do
     {
-      if (sched->ready_count == 0)
+      if (ready_count == 0)
         return;
-      GNUNET_assert (sched->ready[GNUNET_SCHEDULER_PRIORITY_KEEP] == NULL);
+      GNUNET_assert (ready[GNUNET_SCHEDULER_PRIORITY_KEEP] == NULL);
       /* yes, p>0 is correct, 0 is "KEEP" which should
          always be an empty queue (see assertion)! */
       for (p = GNUNET_SCHEDULER_PRIORITY_COUNT - 1; p > 0; p--)
         {
-          pos = sched->ready[p];
+          pos = ready[p];
           if (pos != NULL)
             break;
         }
       GNUNET_assert (pos != NULL);      /* ready_count wrong? */
-      sched->ready[p] = pos->next;
-      sched->ready_count--;
-      sched->current_priority = p;
-      GNUNET_assert (pos->priority == p);
-      tc.sched = sched;
+      ready[p] = pos->next;
+      ready_count--;
+      if (current_priority != pos->priority)
+       {
+         current_priority = pos->priority;
+         (void) GNUNET_OS_set_process_priority (GNUNET_OS_process_current (), pos->priority);
+       }
+      active_task = pos;
+#if PROFILE_DELAYS
+      if (GNUNET_TIME_absolute_get_duration (pos->start_time).rel_value >
+         DELAY_THRESHOLD.rel_value)
+       {
+         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                     "Task %llu took %llums to be scheduled\n",
+                     pos->id,
+                     (unsigned long long) GNUNET_TIME_absolute_get_duration (pos->start_time).rel_value);
+       }
+#endif
       tc.reason = pos->reason;
-      tc.read_ready = pos->read_set;
-      tc.write_ready = pos->write_set;
-      pos->callback (pos->callback_cls, &tc);
+      tc.read_ready = (pos->read_set == NULL) ? rs : pos->read_set; 
+      if ( (pos->read_fd != -1) &&
+          (0 != (pos->reason & GNUNET_SCHEDULER_REASON_READ_READY)) )
+       GNUNET_NETWORK_fdset_set_native (rs,
+                                        pos->read_fd);
+      tc.write_ready = (pos->write_set == NULL) ? ws : pos->write_set;
+      if ( (pos->write_fd != -1) &&
+          (0 != (pos->reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) )
+       GNUNET_NETWORK_fdset_set_native (ws,
+                                        pos->write_fd);
+      if ( ( (tc.reason & GNUNET_SCHEDULER_REASON_WRITE_READY) != 0) &&
+          (pos->write_fd != -1) &&
+          (! GNUNET_NETWORK_fdset_test_native (ws,
+                                               pos->write_fd))) 
+       abort (); // added to ready in previous select loop!
 #if DEBUG_TASKS
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Running task: %llu / %p\n",
-                 pos->id,
-                 pos->callback_cls);
+                  "Running task: %llu / %p\n", pos->id, pos->callback_cls);
+#endif
+      pos->callback (pos->callback_cls, &tc);
+#if EXECINFO
+      int i;
+      for (i=0;i<pos->num_backtrace_strings;i++)
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "Task %llu trace %d: %s\n",
+                    pos->id,
+                    i,
+                    pos->backtrace_strings[i]);
 #endif
+      active_task = NULL;
       destroy_task (pos);
-      sched->tasks_run++;
+      tasks_run++;
     }
-  while ((sched->pending == NULL) || (p == GNUNET_SCHEDULER_PRIORITY_URGENT));
+  while ( (pending == NULL) || (p >= max_priority_added) );
 }
 
-
 /**
  * Pipe used to communicate shutdown via signal.
  */
-static struct GNUNET_DISK_PipeHandle *sigpipe;
-
+static struct GNUNET_DISK_PipeHandle *shutdown_pipe_handle;
 
+/**
+ * Signal handler called for SIGPIPE.
+ */
+#ifndef MINGW
+static void
+sighandler_pipe ()
+{
+  return;
+}
+#endif
 /**
  * Signal handler called for signals that should cause us to shutdown.
  */
@@ -476,143 +680,153 @@ static void
 sighandler_shutdown ()
 {
   static char c;
+  int old_errno = errno; /* backup errno */
 
-  GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_WRITE),
-                         &c,
-                         sizeof(c));
+  GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle
+                          (shutdown_pipe_handle, GNUNET_DISK_PIPE_END_WRITE), &c,
+                          sizeof (c));
+  errno = old_errno;
 }
 
 
 /**
- * Initialize a scheduler using this thread.  This function will
- * return when either a shutdown was initiated (via signal) and all
- * tasks marked to "run_on_shutdown" have been completed or when all
- * tasks in general have been completed.
+ * Initialize and run scheduler.  This function will return when all
+ * tasks have completed.  On systems with signals, receiving a SIGTERM
+ * (and other similar signals) will cause "GNUNET_SCHEDULER_shutdown"
+ * to be run after the active task is complete.  As a result, SIGTERM
+ * causes all active tasks to be scheduled with reason
+ * "GNUNET_SCHEDULER_REASON_SHUTDOWN".  (However, tasks added
+ * afterwards will execute normally!). Note that any particular signal
+ * will only shut down one scheduler; applications should always only
+ * create a single scheduler.
  *
  * @param task task to run immediately
- * @param cls closure of task
+ * @param task_cls closure of task
  */
 void
-GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_Task task, void *cls)
+GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_Task task, void *task_cls)
 {
-  struct GNUNET_SCHEDULER_Handle sched;
   struct GNUNET_NETWORK_FDSet *rs;
   struct GNUNET_NETWORK_FDSet *ws;
   struct GNUNET_TIME_Relative timeout;
   int ret;
   struct GNUNET_SIGNAL_Context *shc_int;
   struct GNUNET_SIGNAL_Context *shc_term;
+#ifndef MINGW
   struct GNUNET_SIGNAL_Context *shc_quit;
   struct GNUNET_SIGNAL_Context *shc_hup;
-  struct Task *tpos;
+  struct GNUNET_SIGNAL_Context *shc_pipe;
+#endif
   unsigned long long last_tr;
   unsigned int busy_wait_warning;
-#ifndef MINGW
   const struct GNUNET_DISK_FileHandle *pr;
-#endif
+  char c;
 
+  GNUNET_assert (active_task == NULL);
   rs = GNUNET_NETWORK_fdset_create ();
   ws = GNUNET_NETWORK_fdset_create ();
-#ifndef MINGW
-  sigpipe = GNUNET_DISK_pipe (GNUNET_NO);
-  pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
+  GNUNET_assert (shutdown_pipe_handle == NULL);
+  shutdown_pipe_handle =  GNUNET_DISK_pipe (GNUNET_NO, GNUNET_NO, GNUNET_NO);
+  GNUNET_assert (shutdown_pipe_handle != NULL);
+  pr = GNUNET_DISK_pipe_handle (shutdown_pipe_handle, GNUNET_DISK_PIPE_END_READ);
+  GNUNET_assert (pr != NULL);
   shc_int = GNUNET_SIGNAL_handler_install (SIGINT, &sighandler_shutdown);
   shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, &sighandler_shutdown);
+#ifndef MINGW
+  shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, &sighandler_pipe);
   shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, &sighandler_shutdown);
   shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, &sighandler_shutdown);
 #endif
-  memset (&sched, 0, sizeof (sched));
-  sched.current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT;
-  GNUNET_SCHEDULER_add_continuation (&sched,
-                                     GNUNET_YES,
-                                     task,
-                                     cls, GNUNET_SCHEDULER_REASON_STARTUP);
+  current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT;
+  GNUNET_SCHEDULER_add_continuation (task,
+                                     task_cls,
+                                     GNUNET_SCHEDULER_REASON_STARTUP);
   last_tr = 0;
   busy_wait_warning = 0;
-  while ((GNUNET_NO == sched.shutdown) &&
-         ((sched.pending != NULL) || (sched.ready_count > 0)))
+  while ((pending != NULL) ||
+        (pending_timeout != NULL) ||
+        (ready_count > 0))
     {
       GNUNET_NETWORK_fdset_zero (rs);
       GNUNET_NETWORK_fdset_zero (ws);
-      timeout = GNUNET_TIME_relative_get_forever();
-      update_sets (&sched, rs, ws, &timeout);
-#ifndef MINGW
+      timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+      update_sets (rs, ws, &timeout);
       GNUNET_NETWORK_fdset_handle_set (rs, pr);
-#endif
-      if (sched.ready_count > 0)
+      if (ready_count > 0)
         {
           /* no blocking, more work already ready! */
-          timeout = GNUNET_TIME_relative_get_zero();
+          timeout = GNUNET_TIME_UNIT_ZERO;
         }
       ret = GNUNET_NETWORK_socket_select (rs, ws, NULL, timeout);
-#ifndef MINGW
-      if (GNUNET_NETWORK_fdset_handle_isset (rs, pr))
-       break;  
-#endif
-      if (last_tr == sched.tasks_run)
-       {
-         busy_wait_warning++;
-       }
-      else
-       {
-         last_tr = sched.tasks_run;
-         busy_wait_warning = 0;
-       }
-      if ( (ret == 0) &&
-          (timeout.value == 0) &&
-          (busy_wait_warning > 16) )
-       {
-         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                     _("Looks like we're busy waiting...\n"));
-         sleep (1); /* mitigate */
-       }
       if (ret == GNUNET_SYSERR)
         {
           if (errno == EINTR)
             continue;
+
           GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "select");
-          break;
+#ifndef MINGW
+#if USE_LSOF
+         char lsof[512];
+         snprintf (lsof, sizeof (lsof), "lsof -p %d", getpid());
+         close (1);
+         dup2 (2, 1);
+         ret = system (lsof);            
+#endif
+#endif
+          abort ();
+         break;
+        }
+      if ((ret == 0) && (timeout.rel_value == 0) && (busy_wait_warning > 16))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                      _("Looks like we're busy waiting...\n"));
+          sleep (1);            /* mitigate */
+        }
+      check_ready (rs, ws);
+      run_ready (rs, ws);
+      if (GNUNET_NETWORK_fdset_handle_isset (rs, pr))
+        {
+          /* consume the signal */
+          GNUNET_DISK_file_read (pr, &c, sizeof (c));
+          /* mark all active tasks as ready due to shutdown */
+          GNUNET_SCHEDULER_shutdown ();
+        }
+      if (last_tr == tasks_run)
+        {
+          busy_wait_warning++;
+        }
+      else
+        {
+          last_tr = tasks_run;
+          busy_wait_warning = 0;
         }
-      check_ready (&sched, rs, ws);
-      run_ready (&sched);
     }
-  sched.shutdown = GNUNET_YES;
-#ifndef MINGW
   GNUNET_SIGNAL_handler_uninstall (shc_int);
   GNUNET_SIGNAL_handler_uninstall (shc_term);
+#ifndef MINGW
+  GNUNET_SIGNAL_handler_uninstall (shc_pipe);
   GNUNET_SIGNAL_handler_uninstall (shc_quit);
   GNUNET_SIGNAL_handler_uninstall (shc_hup);
-  GNUNET_DISK_pipe_close (sigpipe);
-  sigpipe = NULL;
 #endif
-  do
-    {
-      run_ready (&sched);
-      check_ready (&sched, NULL, NULL);
-    }
-  while (sched.ready_count > 0);
-  while (NULL != (tpos = sched.pending))
-    {
-      sched.pending = tpos->next;
-      GNUNET_free (tpos);
-    }
+  GNUNET_DISK_pipe_close (shutdown_pipe_handle);
+  shutdown_pipe_handle = NULL;
   GNUNET_NETWORK_fdset_destroy (rs);
   GNUNET_NETWORK_fdset_destroy (ws);
 }
 
 
 /**
- * Request the shutdown of a scheduler.  This function can be used to
- * stop a scheduling thread when created with the
- * "GNUNET_SCHEDULER_init_thread" function or from within the signal
- * handler for signals causing shutdowns.
+ * Obtain the reason code for why the current task was
+ * started.  Will return the same value as 
+ * the GNUNET_SCHEDULER_TaskContext's reason field.
  *
- * @param sched the scheduler
+ * @return reason(s) why the current task is run
  */
-void
-GNUNET_SCHEDULER_shutdown (struct GNUNET_SCHEDULER_Handle *sched)
+enum GNUNET_SCHEDULER_Reason
+GNUNET_SCHEDULER_get_reason ()
 {
-  sched->shutdown = GNUNET_YES;
+  GNUNET_assert (active_task != NULL);
+  return active_task->reason;
 }
 
 
@@ -622,23 +836,22 @@ GNUNET_SCHEDULER_shutdown (struct GNUNET_SCHEDULER_Handle *sched)
  * dropped (if the decision should be made based on the number of
  * tasks ready to run).
  *
- * @param sched scheduler to query
  * @param p priority level to look at
  * @return number of tasks pending right now
  */
 unsigned int
-GNUNET_SCHEDULER_get_load (struct GNUNET_SCHEDULER_Handle *sched,
-                           enum GNUNET_SCHEDULER_Priority p)
+GNUNET_SCHEDULER_get_load (enum GNUNET_SCHEDULER_Priority p)
 {
   struct Task *pos;
   unsigned int ret;
 
+  GNUNET_assert (active_task != NULL);
   if (p == GNUNET_SCHEDULER_PRIORITY_COUNT)
-    return sched->ready_count;
+    return ready_count;
   if (p == GNUNET_SCHEDULER_PRIORITY_KEEP)
-    p = sched->current_priority;
+    p = current_priority;
   ret = 0;
-  pos = sched->ready[p];
+  pos = ready[check_priority (p)];
   while (pos != NULL)
     {
       pos = pos->next;
@@ -652,21 +865,22 @@ GNUNET_SCHEDULER_get_load (struct GNUNET_SCHEDULER_Handle *sched,
  * Cancel the task with the specified identifier.
  * The task must not yet have run.
  *
- * @param sched scheduler to use
  * @param task id of the task to cancel
  * @return original closure of the task
  */
 void *
-GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
-                         GNUNET_SCHEDULER_TaskIdentifier task)
+GNUNET_SCHEDULER_cancel (GNUNET_SCHEDULER_TaskIdentifier task)
 {
   struct Task *t;
   struct Task *prev;
   enum GNUNET_SCHEDULER_Priority p;
+  int to;
   void *ret;
 
+  GNUNET_assert (active_task != NULL);
+  to = 0;
   prev = NULL;
-  t = sched->pending;
+  t = pending;
   while (t != NULL)
     {
       if (t->id == task)
@@ -674,18 +888,33 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
       prev = t;
       t = t->next;
     }
+  if (t == NULL)
+    {
+      prev = NULL;
+      to = 1;
+      t = pending_timeout;
+      while (t != NULL)
+       {
+         if (t->id == task)
+           break;
+         prev = t;
+         t = t->next;
+       }
+      if (pending_timeout_last == t)
+       pending_timeout_last = NULL;
+    }
   p = 0;
   while (t == NULL)
     {
       p++;
       GNUNET_assert (p < GNUNET_SCHEDULER_PRIORITY_COUNT);
       prev = NULL;
-      t = sched->ready[p];
+      t = ready[p];
       while (t != NULL)
         {
           if (t->id == task)
             {
-              sched->ready_count--;
+              ready_count--;
               break;
             }
           prev = t;
@@ -695,18 +924,29 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
   if (prev == NULL)
     {
       if (p == 0)
-        sched->pending = t->next;
+       {
+         if (to == 0)
+           {
+             pending = t->next;
+           }
+         else
+           {
+             pending_timeout = t->next;
+           }
+       }
       else
-        sched->ready[p] = t->next;
+       {
+         ready[p] = t->next;
+       }
     }
   else
-    prev->next = t->next;
+    {
+      prev->next = t->next;
+    }
   ret = t->callback_cls;
 #if DEBUG_TASKS
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Canceling task: %llu / %p\n",
-             task,
-             t->callback_cls);
+              "Canceling task: %llu / %p\n", task, t->callback_cls);
 #endif
   destroy_task (t);
   return ret;
@@ -718,150 +958,396 @@ GNUNET_SCHEDULER_cancel (struct GNUNET_SCHEDULER_Handle *sched,
  * similar to the other "add" functions except that there is no delay
  * and the reason code can be specified.
  *
- * @param sched scheduler to use
- * @param run_on_shutdown should this task be run if we are shutting down?
- * @param main main function of the task
- * @param cls closure for 'main'
+ * @param task main function of the task
+ * @param task_cls closure for 'main'
  * @param reason reason for task invocation
  */
 void
-GNUNET_SCHEDULER_add_continuation (struct GNUNET_SCHEDULER_Handle *sched,
-                                   int run_on_shutdown,
-                                   GNUNET_SCHEDULER_Task main,
-                                   void *cls,
+GNUNET_SCHEDULER_add_continuation (GNUNET_SCHEDULER_Task task,
+                                   void *task_cls,
                                    enum GNUNET_SCHEDULER_Reason reason)
 {
-  struct Task *task;
+  struct Task *t;
+#if EXECINFO
+  void *backtrace_array[50];
+#endif
 
-  task = GNUNET_malloc (sizeof (struct Task));
-  task->callback = main;
-  task->callback_cls = cls;
-  task->id = ++sched->last_id;
-  task->reason = reason;
-  task->priority = sched->current_priority;
-  task->run_on_shutdown = run_on_shutdown;
+  GNUNET_assert ( (active_task != NULL) ||
+                 (reason == GNUNET_SCHEDULER_REASON_STARTUP) );
+  t = GNUNET_malloc (sizeof (struct Task));
+#if EXECINFO
+  t->num_backtrace_strings = backtrace(backtrace_array, 50);
+  t->backtrace_strings = backtrace_symbols(backtrace_array, t->num_backtrace_strings);
+#endif
+  t->read_fd = -1;
+  t->write_fd = -1;
+  t->callback = task;
+  t->callback_cls = task_cls;
+  t->id = ++last_id;
+#if PROFILE_DELAYS
+  t->start_time = GNUNET_TIME_absolute_get ();
+#endif
+  t->reason = reason;
+  t->priority = current_priority;
 #if DEBUG_TASKS
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Adding continuation task: %llu / %p\n",
-             task->id,
-             task->callback_cls);
+              "Adding continuation task: %llu / %p\n",
+              t->id, t->callback_cls);
 #endif
-  queue_ready_task (sched, task);
+  queue_ready_task (t);
 }
 
 
+
 /**
- * Schedule a new task to be run after the specified
- * prerequisite task has completed.
+ * Schedule a new task to be run after the specified prerequisite task
+ * has completed. It will be run with the priority of the calling
+ * task.
  *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown?
- * @param prio how important is this task?
  * @param prerequisite_task run this task after the task with the given
  *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
+ *        conditions, such as delay, read or write-readiness
  *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param main main function of the task
- * @param cls closure for 'main'
+ *        on completion of other tasks (this will cause the task to run as
+ *        soon as possible).
+ * @param task main function of the task
+ * @param task_cls closure of task
  * @return unique task identifier for the job
- *         only valid until "main" is started!
+ *         only valid until "task" is started!
  */
 GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_after (struct GNUNET_SCHEDULER_Handle *sched,
-                            int run_on_shutdown,
-                            enum GNUNET_SCHEDULER_Priority prio,
-                            GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
-                            GNUNET_SCHEDULER_Task main, void *cls)
+GNUNET_SCHEDULER_add_after (GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
+                            GNUNET_SCHEDULER_Task task, void *task_cls)
 {
-  return GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
+  return GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_KEEP,
                                       prerequisite_task,
                                       GNUNET_TIME_UNIT_ZERO,
-                                      NULL, NULL, main, cls);
+                                      NULL, NULL, task, task_cls);
 }
 
 
+/**
+ * Schedule a new task to be run with a specified priority.
+ *
+ * @param prio how important is the new task?
+ * @param task main function of the task
+ * @param task_cls closure of task
+ * @return unique task identifier for the job
+ *         only valid until "task" is started!
+ */
+GNUNET_SCHEDULER_TaskIdentifier
+GNUNET_SCHEDULER_add_with_priority (enum GNUNET_SCHEDULER_Priority prio,
+                                    GNUNET_SCHEDULER_Task task,
+                                    void *task_cls)
+{
+  return GNUNET_SCHEDULER_add_select (prio,
+                                      GNUNET_SCHEDULER_NO_TASK,
+                                      GNUNET_TIME_UNIT_ZERO,
+                                      NULL, NULL, task, task_cls);
+}
+
+
+
 /**
  * Schedule a new task to be run with a specified delay.  The task
- * will be scheduled for execution once the delay has expired and the
- * prerequisite task has completed.
+ * will be scheduled for execution once the delay has expired. It
+ * will be run with the priority of the calling task.
  *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown? You can use this
- *        argument to run a function only during shutdown
- *        by setting delay to -1.  Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
- * @param prio how important is this task?
- * @param prerequisite_task run this task after the task with the given
- *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
- *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param delay how long should we wait? Use  GNUNET_TIME_UNIT_FOREVER_REL for "forever"
- * @param main main function of the task
- * @param cls closure of task
+ * @param delay when should this operation time out? Use 
+ *        GNUNET_TIME_UNIT_FOREVER_REL for "on shutdown"
+ * @param task main function of the task
+ * @param task_cls closure of task
+ * @return unique task identifier for the job
+ *         only valid until "task" is started!
+ */
+GNUNET_SCHEDULER_TaskIdentifier
+GNUNET_SCHEDULER_add_delayed (struct GNUNET_TIME_Relative delay,
+                              GNUNET_SCHEDULER_Task task, void *task_cls)
+{
+#if 1
+  /* new, optimized version */
+  struct Task *t;
+  struct Task *pos;
+  struct Task *prev;
+#if EXECINFO
+  void *backtrace_array[MAX_TRACE_DEPTH];
+#endif
+
+  GNUNET_assert (active_task != NULL);
+  GNUNET_assert (NULL != task);
+  t = GNUNET_malloc (sizeof (struct Task));
+  t->callback = task;
+  t->callback_cls = task_cls;
+#if EXECINFO
+  t->num_backtrace_strings = backtrace(backtrace_array, MAX_TRACE_DEPTH);
+  t->backtrace_strings = backtrace_symbols(backtrace_array, t->num_backtrace_strings);
+#endif
+  t->read_fd = -1;
+  t->write_fd = -1;
+  t->id = ++last_id;
+#if PROFILE_DELAYS
+  t->start_time = GNUNET_TIME_absolute_get ();
+#endif
+  t->timeout = GNUNET_TIME_relative_to_absolute (delay);
+  t->priority = current_priority;
+  /* try tail first (optimization in case we are
+     appending to a long list of tasks with timeouts) */
+  prev = pending_timeout_last;
+  if (prev != NULL) 
+    {
+      if (prev->timeout.abs_value > t->timeout.abs_value)
+       prev = NULL;
+      else
+       pos = prev->next; /* heuristic success! */
+    }
+  if (prev == NULL)
+    {
+      /* heuristic failed, do traversal of timeout list */
+      pos = pending_timeout;
+    }
+  while ( (pos != NULL) &&
+         ( (pos->timeout.abs_value <= t->timeout.abs_value) ||
+           (pos->reason != 0) ) )
+    {
+      prev = pos;
+      pos = pos->next;
+    }
+  if (prev == NULL)
+    pending_timeout = t;
+  else
+    prev->next = t;
+  t->next = pos;
+  /* hyper-optimization... */
+  pending_timeout_last = t;
+
+#if DEBUG_TASKS
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Adding task: %llu / %p\n", t->id, t->callback_cls);
+#endif
+#if EXECINFO
+  int i;
+
+  for (i=0;i<t->num_backtrace_strings;i++)
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Task %llu trace %d: %s\n",
+                  t->id,
+                  i,
+                  t->backtrace_strings[i]);
+#endif
+  return t->id;
+
+#else
+  /* unoptimized version */
+  return GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_KEEP,
+                                      GNUNET_SCHEDULER_NO_TASK, delay,
+                                      NULL, NULL, task, task_cls);
+#endif
+}
+
+
+
+/**
+ * Schedule a new task to be run as soon as possible. The task
+ * will be run with the priority of the calling task.
+ *
+ * @param task main function of the task
+ * @param task_cls closure of task
+ * @return unique task identifier for the job
+ *         only valid until "task" is started!
+ */
+GNUNET_SCHEDULER_TaskIdentifier
+GNUNET_SCHEDULER_add_now (GNUNET_SCHEDULER_Task task,
+                                                 void *task_cls)
+{
+  return GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_KEEP,
+                                      GNUNET_SCHEDULER_NO_TASK,
+                                     GNUNET_TIME_UNIT_ZERO,
+                                      NULL, NULL, task, task_cls);
+}
+
+
+
+
+/**
+ * Schedule a new task to be run with a specified delay or when any of
+ * the specified file descriptor sets is ready.  The delay can be used
+ * as a timeout on the socket(s) being ready.  The task will be
+ * scheduled for execution once either the delay has expired or any of
+ * the socket operations is ready.  This is the most general
+ * function of the "add" family.  Note that the "prerequisite_task"
+ * must be satisfied in addition to any of the other conditions.  In
+ * other words, the task will be started when
+ * <code>
+ * (prerequisite-run)
+ * && (delay-ready
+ *     || any-rs-ready
+ *     || any-ws-ready
+ *     || shutdown-active )
+ * </code>
+ *
+ * @param delay how long should we wait? Use GNUNET_TIME_UNIT_FOREVER_REL for "forever",
+ *        which means that the task will only be run after we receive SIGTERM
+ * @param rfd file descriptor we want to read (can be -1)
+ * @param wfd file descriptors we want to write (can be -1)
+ * @param task main function of the task
+ * @param task_cls closure of task
  * @return unique task identifier for the job
- *         only valid until "main" is started!
+ *         only valid until "task" is started!
  */
 GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_delayed (struct GNUNET_SCHEDULER_Handle * sched,
-                              int run_on_shutdown,
-                              enum GNUNET_SCHEDULER_Priority prio,
-                              GNUNET_SCHEDULER_TaskIdentifier
-                              prerequisite_task,
-                              struct GNUNET_TIME_Relative delay,
-                              GNUNET_SCHEDULER_Task main, void *cls)
+add_without_sets (struct GNUNET_TIME_Relative delay,
+                 int rfd,
+                 int wfd,
+                 GNUNET_SCHEDULER_Task task, void *task_cls)
 {
-  return GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
-                                      prerequisite_task, delay,
-                                      NULL, NULL, main, cls);
+  struct Task *t;
+#if EXECINFO
+  void *backtrace_array[MAX_TRACE_DEPTH];
+#endif
+
+  GNUNET_assert (active_task != NULL);
+  GNUNET_assert (NULL != task);
+  t = GNUNET_malloc (sizeof (struct Task));
+  t->callback = task;
+  t->callback_cls = task_cls;
+#if EXECINFO
+  t->num_backtrace_strings = backtrace(backtrace_array, MAX_TRACE_DEPTH);
+  t->backtrace_strings = backtrace_symbols(backtrace_array, t->num_backtrace_strings);
+#endif
+  t->read_fd = rfd;
+  t->write_fd = wfd;
+  t->id = ++last_id;
+#if PROFILE_DELAYS
+  t->start_time = GNUNET_TIME_absolute_get ();
+#endif
+  t->prereq_id = GNUNET_SCHEDULER_NO_TASK;
+  t->timeout = GNUNET_TIME_relative_to_absolute (delay);
+  t->priority = check_priority (current_priority);
+  t->next = pending;
+  pending = t;
+  max_priority_added = GNUNET_MAX (max_priority_added,
+                                         t->priority);
+#if DEBUG_TASKS
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Adding task: %llu / %p\n", t->id, t->callback_cls);
+#endif
+#if EXECINFO
+  int i;
+
+  for (i=0;i<t->num_backtrace_strings;i++)
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Task %llu trace %d: %s\n",
+                  t->id,
+                  i,
+                  t->backtrace_strings[i]);
+#endif
+  return t->id;
 }
 
 
+
 /**
  * Schedule a new task to be run with a specified delay or when the
  * specified file descriptor is ready for reading.  The delay can be
  * used as a timeout on the socket being ready.  The task will be
  * scheduled for execution once either the delay has expired or the
- * socket operation is ready.
+ * socket operation is ready.  It will be run with the priority of
+ * the calling task.
  *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown? Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
- * @param prio how important is this task?
- * @param prerequisite_task run this task after the task with the given
- *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
- *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param delay how long should we wait? Use  GNUNET_TIME_UNIT_FOREVER_REL for "forever"
+ * @param delay when should this operation time out? Use 
+ *        GNUNET_TIME_UNIT_FOREVER_REL for "on shutdown"
  * @param rfd read file-descriptor
- * @param main main function of the task
- * @param cls closure of task
+ * @param task main function of the task
+ * @param task_cls closure of task
  * @return unique task identifier for the job
- *         only valid until "main" is started!
+ *         only valid until "task" is started!
  */
 GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_read_net (struct GNUNET_SCHEDULER_Handle * sched,
-                           int run_on_shutdown,
-                           enum GNUNET_SCHEDULER_Priority prio,
-                           GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
-                           struct GNUNET_TIME_Relative delay,
-                           struct GNUNET_NETWORK_Handle *rfd, GNUNET_SCHEDULER_Task main, void *cls)
+GNUNET_SCHEDULER_add_read_net (struct GNUNET_TIME_Relative delay,
+                               struct GNUNET_NETWORK_Handle * rfd,
+                               GNUNET_SCHEDULER_Task task, void *task_cls)
 {
+  return add_without_sets (delay,
+                          GNUNET_NETWORK_get_fd (rfd),
+                          -1,
+                          task,
+                          task_cls);
+}
+
+
+/**
+ * Schedule a new task to be run with a specified delay or when the
+ * specified file descriptor is ready for writing.  The delay can be
+ * used as a timeout on the socket being ready.  The task will be
+ * scheduled for execution once either the delay has expired or the
+ * socket operation is ready.  It will be run with the priority of
+ * the calling task.
+ *
+ * @param delay when should this operation time out? Use 
+ *        GNUNET_TIME_UNIT_FOREVER_REL for "on shutdown"
+ * @param wfd write file-descriptor
+ * @param task main function of the task
+ * @param task_cls closure of task
+ * @return unique task identifier for the job
+ *         only valid until "task" is started!
+ */
+GNUNET_SCHEDULER_TaskIdentifier
+GNUNET_SCHEDULER_add_write_net (struct GNUNET_TIME_Relative delay,
+                                struct GNUNET_NETWORK_Handle * wfd,
+                                GNUNET_SCHEDULER_Task task, void *task_cls)
+{
+  return add_without_sets (delay,
+                          -1,
+                          GNUNET_NETWORK_get_fd (wfd),
+                          task,
+                          task_cls);
+}
+
+
+/**
+ * Schedule a new task to be run with a specified delay or when the
+ * specified file descriptor is ready for reading.  The delay can be
+ * used as a timeout on the socket being ready.  The task will be
+ * scheduled for execution once either the delay has expired or the
+ * socket operation is ready. It will be run with the priority of
+ * the calling task.
+ *
+ * @param delay when should this operation time out? Use 
+ *        GNUNET_TIME_UNIT_FOREVER_REL for "on shutdown"
+ * @param rfd read file-descriptor
+ * @param task main function of the task
+ * @param task_cls closure of task
+ * @return unique task identifier for the job
+ *         only valid until "task" is started!
+ */
+GNUNET_SCHEDULER_TaskIdentifier
+GNUNET_SCHEDULER_add_read_file (struct GNUNET_TIME_Relative delay,
+                                const struct GNUNET_DISK_FileHandle * rfd,
+                                GNUNET_SCHEDULER_Task task, void *task_cls)
+{
+#if MINGW
   struct GNUNET_NETWORK_FDSet *rs;
   GNUNET_SCHEDULER_TaskIdentifier ret;
 
   GNUNET_assert (rfd != NULL);
   rs = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_set (rs, rfd);
-  ret = GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
-                                      prerequisite_task, delay,
-                                      rs, NULL, main, cls);
+  GNUNET_NETWORK_fdset_handle_set (rs, rfd);
+  ret = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_KEEP,
+                                     GNUNET_SCHEDULER_NO_TASK, delay,
+                                     rs, NULL, task, task_cls);
   GNUNET_NETWORK_fdset_destroy (rs);
   return ret;
+#else
+  int fd;
+
+  GNUNET_DISK_internal_file_handle_ (rfd, &fd, sizeof (int));
+  return add_without_sets (delay,
+                          fd,
+                          -1,
+                          task,
+                          task_cls);
+
+#endif
 }
 
 
@@ -870,47 +1356,49 @@ GNUNET_SCHEDULER_add_read_net (struct GNUNET_SCHEDULER_Handle * sched,
  * specified file descriptor is ready for writing.  The delay can be
  * used as a timeout on the socket being ready.  The task will be
  * scheduled for execution once either the delay has expired or the
- * socket operation is ready.
+ * socket operation is ready. It will be run with the priority of
+ * the calling task.
  *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown? Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
- * @param prio how important is this task?
- * @param prerequisite_task run this task after the task with the given
- *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
- *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param delay how long should we wait? Use  GNUNET_TIME_UNIT_FOREVER_REL for "forever"
+ * @param delay when should this operation time out? Use 
+ *        GNUNET_TIME_UNIT_FOREVER_REL for "on shutdown"
  * @param wfd write file-descriptor
- * @param main main function of the task
- * @param cls closure of task
+ * @param task main function of the task
+ * @param task_cls closure of task
  * @return unique task identifier for the job
- *         only valid until "main" is started!
+ *         only valid until "task" is started!
  */
 GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_write_net (struct GNUNET_SCHEDULER_Handle * sched,
-                            int run_on_shutdown,
-                            enum GNUNET_SCHEDULER_Priority prio,
-                            GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
-                            struct GNUNET_TIME_Relative delay,
-                            struct GNUNET_NETWORK_Handle *wfd, GNUNET_SCHEDULER_Task main, void *cls)
+GNUNET_SCHEDULER_add_write_file (struct GNUNET_TIME_Relative delay,
+                                 const struct GNUNET_DISK_FileHandle * wfd,
+                                 GNUNET_SCHEDULER_Task task, void *task_cls)
 {
+#if MINGW
   struct GNUNET_NETWORK_FDSet *ws;
   GNUNET_SCHEDULER_TaskIdentifier ret;
 
   GNUNET_assert (wfd != NULL);
   ws = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_set (ws, wfd);
-  ret = GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
-                                      prerequisite_task, delay,
-                                      NULL, ws, main, cls);
+  GNUNET_NETWORK_fdset_handle_set (ws, wfd);
+  ret = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_KEEP,
+                                     GNUNET_SCHEDULER_NO_TASK,
+                                     delay, NULL, ws, task, task_cls);
   GNUNET_NETWORK_fdset_destroy (ws);
   return ret;
+#else
+  int fd;
+
+  GNUNET_DISK_internal_file_handle_ (wfd, &fd, sizeof (int));
+  return add_without_sets (delay,
+                          -1,
+                          fd,
+                          task,
+                          task_cls);
+
+#endif
 }
 
 
+
 /**
  * Schedule a new task to be run with a specified delay or when any of
  * the specified file descriptor sets is ready.  The delay can be used
@@ -928,161 +1416,85 @@ GNUNET_SCHEDULER_add_write_net (struct GNUNET_SCHEDULER_Handle * sched,
  *     || (shutdown-active && run-on-shutdown) )
  * </code>
  *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown?  Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
  * @param prio how important is this task?
  * @param prerequisite_task run this task after the task with the given
  *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
+ *        conditions, such as delay, read or write-readiness
  *        are satisfied).  Use GNUNET_SCHEDULER_NO_TASK to not have any dependency
  *        on completion of other tasks.
- * @param delay how long should we wait? Use GNUNET_TIME_UNIT_FOREVER_REL for "forever"
+ * @param delay how long should we wait? Use GNUNET_TIME_UNIT_FOREVER_REL for "forever",
+ *        which means that the task will only be run after we receive SIGTERM
  * @param rs set of file descriptors we want to read (can be NULL)
  * @param ws set of file descriptors we want to write (can be NULL)
- * @param main main function of the task
- * @param cls closure of task
+ * @param task main function of the task
+ * @param task_cls closure of task
  * @return unique task identifier for the job
- *         only valid until "main" is started!
+ *         only valid until "task" is started!
  */
 GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_select (struct GNUNET_SCHEDULER_Handle * sched,
-                             int run_on_shutdown,
-                             enum GNUNET_SCHEDULER_Priority prio,
+GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio,
                              GNUNET_SCHEDULER_TaskIdentifier
                              prerequisite_task,
                              struct GNUNET_TIME_Relative delay,
                              const struct GNUNET_NETWORK_FDSet * rs,
-                            const struct GNUNET_NETWORK_FDSet * ws,
-                             GNUNET_SCHEDULER_Task main, void *cls)
+                             const struct GNUNET_NETWORK_FDSet * ws,
+                             GNUNET_SCHEDULER_Task task, void *task_cls)
 {
-  struct Task *task;
+  struct Task *t;
+#if EXECINFO
+  void *backtrace_array[MAX_TRACE_DEPTH];
+#endif
 
-  task = GNUNET_malloc (sizeof (struct Task));
-  task->callback = main;
-  task->callback_cls = cls;
+  GNUNET_assert (active_task != NULL);
+  GNUNET_assert (NULL != task);
+  t = GNUNET_malloc (sizeof (struct Task));
+  t->callback = task;
+  t->callback_cls = task_cls;
+#if EXECINFO
+  t->num_backtrace_strings = backtrace(backtrace_array, MAX_TRACE_DEPTH);
+  t->backtrace_strings = backtrace_symbols(backtrace_array, t->num_backtrace_strings);
+#endif
+  t->read_fd = -1;
+  t->write_fd = -1;
   if (rs != NULL)
     {
-      task->read_set = GNUNET_NETWORK_fdset_create ();
-      GNUNET_NETWORK_fdset_copy (task->read_set, rs);
+      t->read_set = GNUNET_NETWORK_fdset_create ();
+      GNUNET_NETWORK_fdset_copy (t->read_set, rs);
     }
   if (ws != NULL)
     {
-      task->write_set = GNUNET_NETWORK_fdset_create ();
-      GNUNET_NETWORK_fdset_copy (task->write_set, ws);
+      t->write_set = GNUNET_NETWORK_fdset_create ();
+      GNUNET_NETWORK_fdset_copy (t->write_set, ws);
     }
-  task->id = ++sched->last_id;
-  task->prereq_id = prerequisite_task;
-  task->timeout = GNUNET_TIME_relative_to_absolute (delay);
-  task->priority =
+  t->id = ++last_id;
+#if PROFILE_DELAYS
+  t->start_time = GNUNET_TIME_absolute_get ();
+#endif
+  t->prereq_id = prerequisite_task;
+  t->timeout = GNUNET_TIME_relative_to_absolute (delay);
+  t->priority =
     check_priority ((prio ==
-                     GNUNET_SCHEDULER_PRIORITY_KEEP) ? sched->current_priority
+                     GNUNET_SCHEDULER_PRIORITY_KEEP) ? current_priority
                     : prio);
-  task->run_on_shutdown = run_on_shutdown;
-  task->next = sched->pending;
-  sched->pending = task;
+  t->next = pending;
+  pending = t;
+  max_priority_added = GNUNET_MAX (max_priority_added,
+                                         t->priority);
 #if DEBUG_TASKS
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Adding task: %llu / %p\n",
-             task->id,
-             task->callback_cls);
+              "Adding task: %llu / %p\n", t->id, t->callback_cls);
 #endif
-  return task->id;
-}
+#if EXECINFO
+  int i;
 
-/**
- * Schedule a new task to be run with a specified delay or when the
- * specified file descriptor is ready for reading.  The delay can be
- * used as a timeout on the socket being ready.  The task will be
- * scheduled for execution once either the delay has expired or the
- * socket operation is ready.
- *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown? Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
- * @param prio how important is this task?
- * @param prerequisite_task run this task after the task with the given
- *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
- *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param delay how long should we wait? Use  GNUNET_TIME_UNIT_FOREVER_REL for "forever"
- * @param rfd read file-descriptor
- * @param main main function of the task
- * @param cls closure of task
- * @return unique task identifier for the job
- *         only valid until "main" is started!
- */
-GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_read_file (struct GNUNET_SCHEDULER_Handle * sched,
-                               int run_on_shutdown,
-                               enum GNUNET_SCHEDULER_Priority prio,
-                               GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
-                               struct GNUNET_TIME_Relative delay,
-                               const struct GNUNET_DISK_FileHandle *rfd,
-                               GNUNET_SCHEDULER_Task main, void *cls)
-{
-  struct GNUNET_NETWORK_FDSet *rs;
-  GNUNET_SCHEDULER_TaskIdentifier ret;
-
-  GNUNET_assert (rfd != NULL);
-  rs = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_handle_set (rs, rfd);
-  ret = GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
-                                      prerequisite_task, delay,
-                                      rs, NULL, main, cls);
-  GNUNET_NETWORK_fdset_destroy (rs);
-  return ret;
-}
-
-
-/**
- * Schedule a new task to be run with a specified delay or when the
- * specified file descriptor is ready for writing.  The delay can be
- * used as a timeout on the socket being ready.  The task will be
- * scheduled for execution once either the delay has expired or the
- * socket operation is ready.
- *
- * @param sched scheduler to use
- * @param run_on_shutdown run on shutdown? Set this
- *        argument to GNUNET_NO to skip this task if
- *        the user requested process termination.
- * @param prio how important is this task?
- * @param prerequisite_task run this task after the task with the given
- *        task identifier completes (and any of our other
- *        conditions, such as delay, read or write-readyness
- *        are satisfied).  Use  GNUNET_SCHEDULER_NO_TASK to not have any dependency
- *        on completion of other tasks.
- * @param delay how long should we wait? Use  GNUNET_TIME_UNIT_FOREVER_REL for "forever"
- * @param wfd write file-descriptor
- * @param main main function of the task
- * @param cls closure of task
- * @return unique task identifier for the job
- *         only valid until "main" is started!
- */
-GNUNET_SCHEDULER_TaskIdentifier
-GNUNET_SCHEDULER_add_write_file (struct GNUNET_SCHEDULER_Handle * sched,
-                                int run_on_shutdown,
-                                enum GNUNET_SCHEDULER_Priority prio,
-                                GNUNET_SCHEDULER_TaskIdentifier prerequisite_task,
-                                struct GNUNET_TIME_Relative delay,
-                                const struct GNUNET_DISK_FileHandle *wfd,
-                                GNUNET_SCHEDULER_Task main, void *cls)
-{
-  struct GNUNET_NETWORK_FDSet *ws;
-  GNUNET_SCHEDULER_TaskIdentifier ret;
-
-  GNUNET_assert (wfd != NULL);
-  ws = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_handle_set (ws, wfd);
-  ret = GNUNET_SCHEDULER_add_select (sched, run_on_shutdown, prio,
-                                      prerequisite_task, delay,
-                                      NULL, ws, main, cls);
-  GNUNET_NETWORK_fdset_destroy (ws);
-  return ret;
+  for (i=0;i<t->num_backtrace_strings;i++)
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Task %llu trace %d: %s\n",
+                  t->id,
+                  i,
+                  t->backtrace_strings[i]);
+#endif
+  return t->id;
 }
 
-
 /* end of scheduler.c */