use new connecT API
[oweals/gnunet.git] / src / consensus / gnunet-service-consensus.c
index 2d2725b641e53e90904a2b2f971c4cd6581d79a0..99e7aca978a6c8199f71703aad61203ad3bcced0 100644 (file)
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2012, 2013 Christian Grothoff (and other contributing authors)
+      Copyright (C) 2012, 2013 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
 #include "gnunet_protocols.h"
 #include "gnunet_applications.h"
 #include "gnunet_set_service.h"
+#include "gnunet_statistics_service.h"
 #include "gnunet_consensus_service.h"
 #include "consensus_protocol.h"
 #include "consensus.h"
 
-
 #define ELEMENT_TYPE_CONTESTED_MARKER (GNUNET_CONSENSUS_ELEMENT_TYPE_USER_MAX + 1)
 
 
@@ -55,6 +55,14 @@ enum ReferendumVote
 };
 
 
+enum EarlyStoppingPhase
+{
+  EARLY_STOPPING_NONE = 0,
+  EARLY_STOPPING_ONE_MORE = 1,
+  EARLY_STOPPING_DONE = 2,
+};
+
+
 GNUNET_NETWORK_STRUCT_BEGIN
 
 
@@ -157,6 +165,10 @@ enum SetKind
 {
   SET_KIND_NONE = 0,
   SET_KIND_CURRENT,
+  /**
+   * Last result set from a gradecast
+   */
+  SET_KIND_LAST_GRADECAST,
   SET_KIND_LEADER_PROPOSAL,
   SET_KIND_ECHO_RESULT,
 };
@@ -249,6 +261,9 @@ struct Step
 
   struct ConsensusSession *session;
 
+  /**
+   * Tasks that this step is composed of.
+   */
   struct TaskEntry **tasks;
   unsigned int tasks_len;
   unsigned int tasks_cap;
@@ -293,6 +308,19 @@ struct Step
    * the task, used for debugging.
    */
   char *debug_name;
+
+  /**
+   * When we're doing an early finish, how should this step be
+   * treated?
+   * If GNUNET_YES, the step will be marked as finished
+   * without actually running its tasks.
+   * Otherwise, the step will still be run even after
+   * an early finish.
+   *
+   * Note that a task may never be finished early if
+   * it is already running.
+   */
+  int early_finishable;
 };
 
 
@@ -459,6 +487,11 @@ struct ConsensusSession
    * Uses the session's global id as app id.
    */
   struct GNUNET_SET_ListenHandle *set_listener;
+
+  /**
+   * State of our early stopping scheme.
+   */
+  int early_stopping;
 };
 
 /**
@@ -486,6 +519,11 @@ static struct GNUNET_SERVER_Handle *srv;
  */
 static struct GNUNET_PeerIdentity my_peer;
 
+/**
+ * Statistics handle.
+ */
+struct GNUNET_STATISTICS_Handle *statistics;
+
 
 static void
 finish_task (struct TaskEntry *task);
@@ -793,6 +831,22 @@ rfn_contest (struct ReferendumEntry *rfn,
   rfn->peer_contested[contested_peer] = GNUNET_YES;
 }
 
+
+static uint16_t
+rfn_noncontested (struct ReferendumEntry *rfn)
+{
+  uint16_t i;
+  uint16_t ret;
+
+  ret = 0;
+  for (i = 0; i < rfn->num_peers; i++)
+    if ( (GNUNET_YES == rfn->peer_commited[i]) && (GNUNET_NO == rfn->peer_contested[i]) )
+      ret++;
+
+  return ret;
+}
+
+
 static void
 rfn_vote (struct ReferendumEntry *rfn,
           uint16_t voting_peer,
@@ -860,7 +914,7 @@ set_result_cb (void *cls,
 
   setop = &task->cls.setop;
 
-  
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "P%u: got set result for {%s}, status %u\n",
               session->local_peer_idx,
@@ -879,15 +933,7 @@ set_result_cb (void *cls,
     return;
   }
 
-  if (task->key.peer1 == session->local_peer_idx)
-    other_idx = task->key.peer2;
-  else if (task->key.peer2 == session->local_peer_idx)
-    other_idx = task->key.peer1;
-  else
-  {
-    /* error in task graph construction */
-    GNUNET_assert (0);
-  }
+  other_idx = task_other_peer (task);
 
   if (SET_KIND_NONE != setop->output_set.set_kind)
   {
@@ -909,10 +955,12 @@ set_result_cb (void *cls,
 
   if (GNUNET_YES == session->peers_blacklisted[other_idx])
   {
-    /* We should have never started or commited to an operation
-       with a blacklisted peer. */
-    GNUNET_break (0);
-    return;
+    /* Peer might have been blacklisted
+       by a gradecast running in parallel, ignore elements from now */
+    if (GNUNET_SET_STATUS_ADD_LOCAL == status)
+      return;
+    if (GNUNET_SET_STATUS_ADD_REMOTE == status)
+      return;
   }
 
   if ( (GNUNET_SET_STATUS_ADD_LOCAL == status) || (GNUNET_SET_STATUS_ADD_REMOTE == status) )
@@ -1036,7 +1084,8 @@ set_result_cb (void *cls,
       break;
     case GNUNET_SET_STATUS_FAILURE:
       // XXX: cleanup
-      GNUNET_break (0);
+      GNUNET_break_op (0);
+      finish_task (task);
       return;
     default:
       /* not reached */
@@ -1046,26 +1095,68 @@ set_result_cb (void *cls,
 
 #ifdef EVIL
 
-enum Evilness
+enum EvilnessType
 {
   EVILNESS_NONE,
-  EVILNESS_CRAM,
+  EVILNESS_CRAM_ALL,
+  EVILNESS_CRAM_LEAD,
+  EVILNESS_CRAM_ECHO,
   EVILNESS_SLACK,
 };
 
+enum EvilnessSubType
+{
+  EVILNESS_SUB_NONE,
+  EVILNESS_SUB_REPLACEMENT,
+  EVILNESS_SUB_NO_REPLACEMENT,
+};
+
+struct Evilness
+{
+  enum EvilnessType type;
+  enum EvilnessSubType subtype;
+  unsigned int num;
+};
+
+
+static int
+parse_evilness_cram_subtype (const char *evil_subtype_str, struct Evilness *evil)
+{
+  if (0 == strcmp ("replace", evil_subtype_str))
+  {
+    evil->subtype = EVILNESS_SUB_REPLACEMENT;
+  }
+  else if (0 == strcmp ("noreplace", evil_subtype_str))
+  {
+    evil->subtype = EVILNESS_SUB_NO_REPLACEMENT;
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Malformed field '%s' in EVIL_SPEC (unknown subtype), behaving like a good peer.\n",
+                evil_subtype_str);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
 static void
-get_evilness (struct ConsensusSession *session, enum Evilness *ret_type, unsigned int *ret_num)
+get_evilness (struct ConsensusSession *session, struct Evilness *evil)
 {
   char *evil_spec;
   char *field;
   char *evil_type_str = NULL;
+  char *evil_subtype_str = NULL;
+
+  GNUNET_assert (NULL != evil);
 
   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "consensus", "EVIL_SPEC", &evil_spec))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "P%u: no evilness\n",
                 session->local_peer_idx);
-    *ret_type = EVILNESS_NONE;
+    evil->type = EVILNESS_NONE;
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1081,30 +1172,54 @@ get_evilness (struct ConsensusSession *session, enum Evilness *ret_type, unsigne
     int ret;
 
     evil_type_str = NULL;
+    evil_subtype_str = NULL;
 
-    ret = sscanf (field, "%u;%m[a-z];%u", &peer_num, &evil_type_str, &evil_num);
+    ret = sscanf (field, "%u;%m[a-z-];%m[a-z-];%u", &peer_num, &evil_type_str, &evil_subtype_str, &evil_num);
 
-    if (ret != 3)
+    if (ret != 4)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Malformed field '%s' in EVIL_SPEC, behaving like a good peer.\n",
-                  field); 
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Malformed field '%s' in EVIL_SPEC (expected 4 components got %d), behaving like a good peer.\n",
+                  field,
+                  ret);
       goto not_evil;
     }
 
     GNUNET_assert (NULL != evil_type_str);
+    GNUNET_assert (NULL != evil_subtype_str);
 
     if (peer_num == session->local_peer_idx)
     {
       if (0 == strcmp ("slack", evil_type_str))
-        *ret_type = EVILNESS_SLACK;
-      else if (0 == strcmp ("cram", evil_type_str))
       {
-        *ret_type = EVILNESS_CRAM;
-        *ret_num = evil_num;
+        evil->type = EVILNESS_SLACK;
+      }
+      else if (0 == strcmp ("cram-all", evil_type_str))
+      {
+        evil->type = EVILNESS_CRAM_ALL;
+        evil->num = evil_num;
+        if (GNUNET_OK != parse_evilness_cram_subtype (evil_subtype_str, evil))
+          goto not_evil;
+      }
+      else if (0 == strcmp ("cram-lead", evil_type_str))
+      {
+        evil->type = EVILNESS_CRAM_LEAD;
+        evil->num = evil_num;
+        if (GNUNET_OK != parse_evilness_cram_subtype (evil_subtype_str, evil))
+          goto not_evil;
+      }
+      else if (0 == strcmp ("cram-echo", evil_type_str))
+      {
+        evil->type = EVILNESS_CRAM_ECHO;
+        evil->num = evil_num;
+        if (GNUNET_OK != parse_evilness_cram_subtype (evil_subtype_str, evil))
+          goto not_evil;
       }
       else
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Malformed field '%s' in EVIL_SPEC (unknown type), behaving like a good peer.\n"); 
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "Malformed field '%s' in EVIL_SPEC (unknown type), behaving like a good peer.\n",
+                    evil_type_str);
         goto not_evil;
       }
       goto cleanup;
@@ -1112,13 +1227,18 @@ get_evilness (struct ConsensusSession *session, enum Evilness *ret_type, unsigne
     /* No GNUNET_free since memory was allocated by libc */
     free (evil_type_str);
     evil_type_str = NULL;
+    evil_subtype_str = NULL;
   }
 not_evil:
-  *ret_type = EVILNESS_NONE;
+  evil->type = EVILNESS_NONE;
 cleanup:
   GNUNET_free (evil_spec);
+  /* no GNUNET_free_non_null since it wasn't
+   * allocated with GNUNET_malloc */
   if (NULL != evil_type_str)
     free (evil_type_str);
+  if (NULL != evil_subtype_str)
+    free (evil_subtype_str);
 }
 
 #endif
@@ -1142,13 +1262,22 @@ commit_set (struct ConsensusSession *session,
 #ifdef EVIL
   {
     unsigned int i;
-    unsigned int evil_num;
-    enum Evilness evilness;
+    struct Evilness evil;
 
-    get_evilness (session, &evilness, &evil_num);
-    switch (evilness)
+    get_evilness (session, &evil);
+    if (EVILNESS_NONE != evil.type)
+    {
+      /* Useful for evaluation */
+      GNUNET_STATISTICS_set (statistics,
+                             "is evil",
+                             1,
+                             GNUNET_NO);
+    }
+    switch (evil.type)
     {
-      case EVILNESS_CRAM:
+      case EVILNESS_CRAM_ALL:
+      case EVILNESS_CRAM_LEAD:
+      case EVILNESS_CRAM_ECHO:
         /* We're not cramming elements in the
            all-to-all round, since that would just
            add more elements to the result set, but
@@ -1158,7 +1287,18 @@ commit_set (struct ConsensusSession *session,
           GNUNET_SET_commit (setop->op, set->h);
           break;
         }
-        for (i = 0; i < evil_num; i++)
+        if ((EVILNESS_CRAM_LEAD == evil.type) &&
+            ((PHASE_KIND_GRADECAST_LEADER != task->key.kind) || SET_KIND_CURRENT != set->key.set_kind))
+        {
+          GNUNET_SET_commit (setop->op, set->h);
+          break;
+        }
+        if (EVILNESS_CRAM_ECHO == evil.type && (PHASE_KIND_GRADECAST_ECHO != task->key.kind))
+        {
+          GNUNET_SET_commit (setop->op, set->h);
+          break;
+        }
+        for (i = 0; i < evil.num; i++)
         {
           struct GNUNET_HashCode hash;
           struct GNUNET_SET_Element element;
@@ -1166,7 +1306,20 @@ commit_set (struct ConsensusSession *session,
           element.size = sizeof (struct GNUNET_HashCode);
           element.element_type = 0;
 
-          GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
+          if (EVILNESS_SUB_REPLACEMENT == evil.subtype)
+          {
+            /* Always generate a new element. */
+            GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &hash);
+          }
+          else if (EVILNESS_SUB_NO_REPLACEMENT == evil.subtype)
+          {
+            /* Always cram the same elements, derived from counter. */
+            GNUNET_CRYPTO_hash (&i, sizeof (i), &hash);
+          }
+          else
+          {
+            GNUNET_assert (0);
+          }
           GNUNET_SET_add_element (set->h, &element, NULL, NULL);
 #ifdef GNUNET_EXTRA_LOGGING
           GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -1177,13 +1330,17 @@ commit_set (struct ConsensusSession *session,
                       debug_str_task_key (&task->key));
 #endif
         }
+        GNUNET_STATISTICS_update (statistics,
+                                  "# stuffed elements",
+                                  evil.num,
+                                  GNUNET_NO);
         GNUNET_SET_commit (setop->op, set->h);
         break;
       case EVILNESS_SLACK:
         GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                     "P%u: evil peer: slacking\n",
                     session->local_peer_idx,
-                    evil_num);
+                    evil.num);
         /* Do nothing. */
         break;
       case EVILNESS_NONE:
@@ -1201,7 +1358,17 @@ commit_set (struct ConsensusSession *session,
     element.element_type = ELEMENT_TYPE_CONTESTED_MARKER;
     GNUNET_SET_add_element (set->h, &element, NULL, NULL);
   }
-  GNUNET_SET_commit (setop->op, set->h);
+  if (GNUNET_NO == session->peers_blacklisted[task_other_peer (task)])
+  {
+    GNUNET_SET_commit (setop->op, set->h);
+  }
+  else
+  {
+    /* For our testcases, we don't want the blacklisted
+       peers to wait. */
+    GNUNET_SET_operation_cancel (setop->op);
+    setop->op = NULL;
+  }
 #endif
 }
 
@@ -1233,9 +1400,9 @@ put_set (struct ConsensusSession *session,
               debug_str_set_key (&set->key));
 
   GNUNET_CRYPTO_hash (&set->key, sizeof (struct SetKey), &hash);
-  GNUNET_assert (GNUNET_OK ==
+  GNUNET_assert (GNUNET_SYSERR !=
                  GNUNET_CONTAINER_multihashmap_put (session->setmap, &hash, set,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE));
 }
 
 
@@ -1297,7 +1464,7 @@ diff_create ()
   struct DiffEntry *d = GNUNET_new (struct DiffEntry);
 
   d->changes = GNUNET_CONTAINER_multihashmap_create (8, GNUNET_NO);
-  
+
   return d;
 }
 
@@ -1309,7 +1476,7 @@ diff_compose (struct DiffEntry *diff_1,
   struct DiffEntry *diff_new;
   struct GNUNET_CONTAINER_MultiHashMapIterator *iter;
   struct DiffElementInfo *di;
+
   diff_new = diff_create ();
 
   iter = GNUNET_CONTAINER_multihashmap_iterator_create (diff_1->changes);
@@ -1338,6 +1505,7 @@ rfn_create (uint16_t size)
   rfn = GNUNET_new (struct ReferendumEntry);
   rfn->rfn_elements = GNUNET_CONTAINER_multihashmap_create (8, GNUNET_NO);
   rfn->peer_commited = GNUNET_new_array (size, int);
+  rfn->peer_contested = GNUNET_new_array (size, int);
   rfn->num_peers = size;
 
   return rfn;
@@ -1352,6 +1520,11 @@ diff_destroy (struct DiffEntry *diff)
 }
 
 
+/**
+ * For a given majority, count what the outcome
+ * is (add/remove/keep), and give the number
+ * of peers that voted for this outcome.
+ */
 static void
 rfn_majority (const struct ReferendumEntry *rfn,
               const struct RfnElementInfo *ri,
@@ -1470,6 +1643,89 @@ set_mutation_done (void *cls)
   }
 }
 
+
+static void
+try_finish_step_early (struct Step *step)
+{
+  unsigned int i;
+
+  if (GNUNET_YES == step->is_running)
+    return;
+  if (GNUNET_YES == step->is_finished)
+    return;
+  if (GNUNET_NO == step->early_finishable)
+    return;
+
+  step->is_finished = GNUNET_YES;
+
+#ifdef GNUNET_EXTRA_LOGGING
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finishing step `%s' early.\n",
+              step->debug_name);
+#endif
+
+  for (i = 0; i < step->subordinates_len; i++)
+  {
+    GNUNET_assert (step->subordinates[i]->pending_prereq > 0);
+    step->subordinates[i]->pending_prereq--;
+#ifdef GNUNET_EXTRA_LOGGING
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Decreased pending_prereq to %u for step `%s'.\n",
+                step->subordinates[i]->pending_prereq,
+                step->subordinates[i]->debug_name);
+
+#endif
+    try_finish_step_early (step->subordinates[i]);
+  }
+
+  // XXX: maybe schedule as task to avoid recursion?
+  run_ready_steps (step->session);
+}
+
+
+static void
+finish_step (struct Step *step)
+{
+  unsigned int i;
+
+  GNUNET_assert (step->finished_tasks == step->tasks_len);
+  GNUNET_assert (GNUNET_YES == step->is_running);
+  GNUNET_assert (GNUNET_NO == step->is_finished);
+
+#ifdef GNUNET_EXTRA_LOGGING
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "All tasks of step `%s' with %u subordinates finished.\n",
+              step->debug_name,
+              step->subordinates_len);
+#endif
+
+  for (i = 0; i < step->subordinates_len; i++)
+  {
+    GNUNET_assert (step->subordinates[i]->pending_prereq > 0);
+    step->subordinates[i]->pending_prereq--;
+#ifdef GNUNET_EXTRA_LOGGING
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Decreased pending_prereq to %u for step `%s'.\n",
+                step->subordinates[i]->pending_prereq,
+                step->subordinates[i]->debug_name);
+
+#endif
+  }
+
+  step->is_finished = GNUNET_YES;
+
+  // XXX: maybe schedule as task to avoid recursion?
+  run_ready_steps (step->session);
+}
+
+
+
+/**
+ * Apply the result from one round of gradecasts (i.e. every peer
+ * should have gradecasted) to the peer's current set.
+ *
+ * @param task the task with context information
+ */
 static void
 task_start_apply_round (struct TaskEntry *task)
 {
@@ -1482,6 +1738,7 @@ task_start_apply_round (struct TaskEntry *task)
   struct GNUNET_CONTAINER_MultiHashMapIterator *iter;
   struct RfnElementInfo *ri;
   struct SetMutationProgressCls *progress_cls;
+  uint16_t worst_majority = UINT16_MAX;
 
   sk_in = (struct SetKey) { SET_KIND_CURRENT, task->key.repetition };
   rk_in = (struct RfnKey) { RFN_KIND_GRADECAST_RESULT, task->key.repetition };
@@ -1509,6 +1766,9 @@ task_start_apply_round (struct TaskEntry *task)
 
     rfn_majority (rfn_in, ri, &majority_num, &majority_vote);
 
+    if (worst_majority > majority_num)
+      worst_majority = majority_num;
+
     switch (majority_vote)
     {
       case VOTE_ADD:
@@ -1518,6 +1778,10 @@ task_start_apply_round (struct TaskEntry *task)
                                                ri->element,
                                                set_mutation_done,
                                                progress_cls));
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "P%u: apply round: adding element %s with %u-majority.\n",
+                    session->local_peer_idx,
+                    debug_str_element (ri->element), majority_num);
         break;
       case VOTE_REMOVE:
         progress_cls->num_pending++;
@@ -1526,8 +1790,16 @@ task_start_apply_round (struct TaskEntry *task)
                                                   ri->element,
                                                   set_mutation_done,
                                                   progress_cls));
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "P%u: apply round: deleting element %s with %u-majority.\n",
+                    session->local_peer_idx,
+                    debug_str_element (ri->element), majority_num);
         break;
       case VOTE_STAY:
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "P%u: apply round: keeping element %s with %u-majority.\n",
+                    session->local_peer_idx,
+                    debug_str_element (ri->element), majority_num);
         // do nothing
         break;
       default:
@@ -1542,10 +1814,53 @@ task_start_apply_round (struct TaskEntry *task)
     GNUNET_free (progress_cls);
     finish_task (task);
   }
-}
 
+  {
+    uint16_t thresh = (session->num_peers / 3) * 2;
+
+    if (worst_majority >= thresh)
+    {
+      switch (session->early_stopping)
+      {
+        case EARLY_STOPPING_NONE:
+          session->early_stopping = EARLY_STOPPING_ONE_MORE;
+          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                      "P%u: Stopping early (after one more superround)\n",
+                      session->local_peer_idx);
+          break;
+        case EARLY_STOPPING_ONE_MORE:
+          GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: finishing steps due to early finish\n",
+                      session->local_peer_idx);
+          session->early_stopping = EARLY_STOPPING_DONE;
+          {
+            struct Step *step;
+            for (step = session->steps_head; NULL != step; step = step->next)
+              try_finish_step_early (step);
+          }
+          break;
+        case EARLY_STOPPING_DONE:
+          /* We shouldn't be here anymore after early stopping */
+          GNUNET_break (0);
+          break;
+        default:
+          GNUNET_assert (0);
+          break;
+      }
+    }
+    else if (EARLY_STOPPING_NONE != session->early_stopping)
+    {
+      // Our assumption about the number of bad peers
+      // has been broken.
+      GNUNET_break_op (0);
+    }
+    else
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: NOT finishing early (majority not good enough)\n",
+                  session->local_peer_idx);
+    }
+  }
 
-#define THRESH(s) (((s)->num_peers / 3))
+}
 
 
 static void
@@ -1590,14 +1905,8 @@ task_start_grade (struct TaskEntry *task)
     // XXX: we need contested votes and non-contested votes here
     rfn_majority (input_rfn, ri, &majority_num, &majority_vote);
 
-    if (majority_num < (session->num_peers / 3) * 2)
-    {
-      gradecast_confidence = GNUNET_MIN(1, gradecast_confidence);
-    }
-    if (majority_num < (session->num_peers / 3) + 1)
-    {
-      gradecast_confidence = 0;
-    }
+    if (majority_num <= session->num_peers / 3)
+      majority_vote = VOTE_REMOVE;
 
     switch (majority_vote)
     {
@@ -1615,6 +1924,19 @@ task_start_grade (struct TaskEntry *task)
     }
   }
 
+  {
+    uint16_t noncontested;
+    noncontested = rfn_noncontested (input_rfn);
+    if (noncontested < (session->num_peers / 3) * 2)
+    {
+      gradecast_confidence = GNUNET_MIN(1, gradecast_confidence);
+    }
+    if (noncontested < (session->num_peers / 3) + 1)
+    {
+      gradecast_confidence = 0;
+    }
+  }
+
   if (gradecast_confidence >= 1)
     rfn_commit (output_rfn, task->key.leader);
 
@@ -1719,12 +2041,7 @@ task_start_reconcile (struct TaskEntry *task)
                                     set_result_cb,
                                     task);
 
-    if (GNUNET_OK != GNUNET_SET_commit (setop->op, input->h))
-    {
-      GNUNET_break (0);
-      /* XXX: cleanup? */
-      return;
-    }
+    commit_set (session, task);
   }
   else if (task->key.peer2 == session->local_peer_idx)
   {
@@ -1767,6 +2084,16 @@ task_start_eval_echo (struct TaskEntry *task)
     return;
   }
 
+
+  {
+    // FIXME: should be marked as a shallow copy, so
+    // we can destroy everything correctly
+    struct SetEntry *last_set = GNUNET_new (struct SetEntry);
+    last_set->h = output_set->h;
+    last_set->key = (struct SetKey) { SET_KIND_LAST_GRADECAST };
+    put_set (session, last_set);
+  }
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Evaluating referendum in Task {%s}\n",
               debug_str_task_key (&task->key));
@@ -1872,39 +2199,6 @@ start_task (struct ConsensusSession *session, struct TaskEntry *task)
 }
 
 
-static void finish_step (struct Step *step)
-{
-  unsigned int i;
-
-  GNUNET_assert (step->finished_tasks == step->tasks_len);
-  GNUNET_assert (GNUNET_YES == step->is_running);
-  GNUNET_assert (GNUNET_NO == step->is_finished);
-
-#ifdef GNUNET_EXTRA_LOGGING
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "All tasks of step `%s' with %u subordinates finished.\n",
-              step->debug_name,
-              step->subordinates_len);
-#endif
-
-  for (i = 0; i < step->subordinates_len; i++)
-  {
-    GNUNET_assert (step->subordinates[i]->pending_prereq > 0);
-    step->subordinates[i]->pending_prereq--;
-#ifdef GNUNET_EXTRA_LOGGING
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Decreased pending_prereq to %u for step `%s'.\n",
-                step->subordinates[i]->pending_prereq,
-                step->subordinates[i]->debug_name);
-
-#endif
-  }
-
-  step->is_finished = GNUNET_YES;
-
-  // XXX: maybe schedule as task to avoid recursion?
-  run_ready_steps (step->session);
-}
 
 
 /*
@@ -1920,7 +2214,7 @@ run_ready_steps (struct ConsensusSession *session)
 
   while (NULL != step)
   {
-    if ( (GNUNET_NO == step->is_running) && (0 == step->pending_prereq) )
+    if ( (GNUNET_NO == step->is_running) && (0 == step->pending_prereq) && (GNUNET_NO == step->is_finished) )
     {
       size_t i;
 
@@ -2169,8 +2463,8 @@ set_listen_cb (void *cls,
                                           GNUNET_SET_RESULT_SYMMETRIC,
                                           set_result_cb,
                                           task);
-  
-  /* If the task hasn't been started yet, 
+
+  /* If the task hasn't been started yet,
      we wait for that until we commit. */
 
   if (GNUNET_YES == task->is_started)
@@ -2225,6 +2519,8 @@ install_step_timeouts (struct ConsensusSession *session)
   /* Given the fully constructed task graph
      with rounds for tasks, we can give the tasks timeouts. */
 
+  // unsigned int max_round;
+
   /* XXX: implement! */
 }
 
@@ -2269,7 +2565,7 @@ arrange_peers (uint16_t *p1, uint16_t *p2, uint16_t n)
 
 
 /**
- * Record @a dep as a dependency of @step.
+ * Record @a dep as a dependency of @step.
  */
 static void
 step_depend_on (struct Step *step, struct Step *dep)
@@ -2311,12 +2607,13 @@ step_depend_on (struct Step *step, struct Step *dep)
 
 
 static struct Step *
-create_step (struct ConsensusSession *session, int round)
+create_step (struct ConsensusSession *session, int round, int early_finishable)
 {
   struct Step *step;
   step = GNUNET_new (struct Step);
   step->session = session;
   step->round = round;
+  step->early_finishable = early_finishable;
   GNUNET_CONTAINER_DLL_insert_tail (session->steps_head,
                                     session->steps_tail,
                                     step);
@@ -2355,7 +2652,7 @@ construct_task_graph_gradecast (struct ConsensusSession *session,
 
   /* gcast step 1: leader disseminates */
 
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_YES);
 
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "disseminate leader %u rep %u", lead, rep);
@@ -2415,7 +2712,7 @@ construct_task_graph_gradecast (struct ConsensusSession *session,
   /* gcast phase 2: echo */
   prev_step = step;
   round += 1;
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "echo leader %u rep %u", lead, rep);
 #endif
@@ -2439,7 +2736,7 @@ construct_task_graph_gradecast (struct ConsensusSession *session,
 
   prev_step = step;
   /* Same round, since step only has local tasks */
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "echo grade leader %u rep %u", lead, rep);
 #endif
@@ -2455,7 +2752,7 @@ construct_task_graph_gradecast (struct ConsensusSession *session,
 
   prev_step = step;
   round += 1;
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "confirm leader %u rep %u", lead, rep);
 #endif
@@ -2485,7 +2782,7 @@ construct_task_graph_gradecast (struct ConsensusSession *session,
 
   prev_step = step;
   /* Same round, since step only has local tasks */
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "confirm grade leader %u rep %u", lead, rep);
 #endif
@@ -2529,14 +2826,14 @@ construct_task_graph (struct ConsensusSession *session)
   // XXX: introduce first step,
   // where we wait for all insert acks
   // from the set service
-  
+
   /* faster but brittle all-to-all */
 
   // XXX: Not implemented yet
 
   /* all-to-all step */
 
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_NO);
 
 #ifdef GNUNET_EXTRA_LOGGING
   step->debug_name = GNUNET_strdup ("all to all");
@@ -2573,7 +2870,7 @@ construct_task_graph (struct ConsensusSession *session)
     struct Step *step_rep_end;
 
     /* Every repetition is in a separate round. */
-    step_rep_start = create_step (session, round);
+    step_rep_start = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
     GNUNET_asprintf (&step_rep_start->debug_name, "gradecast start rep %u", i);
 #endif
@@ -2582,7 +2879,7 @@ construct_task_graph (struct ConsensusSession *session)
 
     /* gradecast has three rounds */
     round += 3;
-    step_rep_end = create_step (session, round);
+    step_rep_end = create_step (session, round, GNUNET_YES);
 #ifdef GNUNET_EXTRA_LOGGING
     GNUNET_asprintf (&step_rep_end->debug_name, "gradecast end rep %u", i);
 #endif
@@ -2604,7 +2901,7 @@ construct_task_graph (struct ConsensusSession *session)
  /* There is no next gradecast round, thus the final
     start step is the overall end step of the gradecasts */
   round += 1;
-  step = create_step (session, round);
+  step = create_step (session, round, GNUNET_NO);
 #ifdef GNUNET_EXTRA_LOGGING
   GNUNET_asprintf (&step->debug_name, "finish");
 #endif
@@ -2615,7 +2912,7 @@ construct_task_graph (struct ConsensusSession *session)
     .key = (struct TaskKey) { PHASE_KIND_FINISH, -1, -1, -1, -1 },
     .start = task_start_finish,
   });
-  task.cls.finish.input_set = (struct SetKey) { SET_KIND_CURRENT, t + 1 };
+  task.cls.finish.input_set = (struct SetKey) { SET_KIND_LAST_GRADECAST };
 
   put_task (session->taskmap, &task);
 }
@@ -2872,15 +3169,14 @@ client_conclude (void *cls,
  * Called to clean up, after a shutdown has been requested.
  *
  * @param cls closure
- * @param tc context information (why was this task triggered now)
  */
 static void
-shutdown_task (void *cls,
-               const struct GNUNET_SCHEDULER_TaskContext *tc)
+shutdown_task (void *cls)
 {
   while (NULL != sessions_head)
     destroy_session (sessions_head);
 
+  GNUNET_STATISTICS_destroy (statistics, GNUNET_YES);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handled shutdown request\n");
 }
 
@@ -2933,8 +3229,9 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
+  statistics = GNUNET_STATISTICS_create ("consensus", cfg);
   GNUNET_SERVER_add_handlers (server, server_handlers);
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
   GNUNET_SERVER_disconnect_notify (server, handle_client_disconnect, NULL);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "consensus running\n");
 }
@@ -2955,4 +3252,3 @@ main (int argc, char *const *argv)
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "exit (%d)\n", GNUNET_OK != ret);
   return (GNUNET_OK == ret) ? 0 : 1;
 }
-