Merge branch 'master' of ssh://gnunet.org/gnunet
[oweals/gnunet.git] / src / consensus / gnunet-service-consensus.c
index ad01a276bb60f2f6818998a8add3d16d8f9e31f2..b934f468fbddad7cb6ea1a635e9f6175c5b147dd 100644 (file)
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2012, 2013 GNUnet e.V.
+      Copyright (C) 2012, 2013, 2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -26,6 +26,7 @@
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_block_lib.h"
 #include "gnunet_protocols.h"
 #include "gnunet_applications.h"
 #include "gnunet_set_service.h"
@@ -34,8 +35,6 @@
 #include "consensus_protocol.h"
 #include "consensus.h"
 
-#define ELEMENT_TYPE_CONTESTED_MARKER (GNUNET_CONSENSUS_ELEMENT_TYPE_USER_MAX + 1)
-
 
 enum ReferendumVote
 {
@@ -65,11 +64,6 @@ enum EarlyStoppingPhase
 
 GNUNET_NETWORK_STRUCT_BEGIN
 
-
-struct ContestedPayload
-{
-};
-
 /**
  * Tuple of integers that together
  * identify a task uniquely.
@@ -147,6 +141,7 @@ GNUNET_NETWORK_STRUCT_END
 enum PhaseKind
 {
   PHASE_KIND_ALL_TO_ALL,
+  PHASE_KIND_ALL_TO_ALL_2,
   PHASE_KIND_GRADECAST_LEADER,
   PHASE_KIND_GRADECAST_ECHO,
   PHASE_KIND_GRADECAST_ECHO_GRADE,
@@ -451,7 +446,7 @@ struct ConsensusSession
   /**
    * Client that inhabits the session
    */
-  struct GNUNET_SERVER_Client *client;
+  struct GNUNET_SERVICE_Client *client;
 
   /**
    * Queued messages to the client.
@@ -492,6 +487,18 @@ struct ConsensusSession
    * State of our early stopping scheme.
    */
   int early_stopping;
+
+  /**
+   * Our set size from the first round.
+   */
+  uint64_t first_size;
+
+  uint64_t *first_sizes_received;
+
+  /**
+   * Bounded Eppstein lower bound.
+   */
+  uint64_t lower_bound;
 };
 
 /**
@@ -509,11 +516,6 @@ static struct ConsensusSession *sessions_tail;
  */
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
-/**
- * Handle to the server for this service.
- */
-static struct GNUNET_SERVER_Handle *srv;
-
 /**
  * Peer that runs this service.
  */
@@ -528,15 +530,18 @@ struct GNUNET_STATISTICS_Handle *statistics;
 static void
 finish_task (struct TaskEntry *task);
 
+
 static void
 run_ready_steps (struct ConsensusSession *session);
 
+
 static const char *
 phasename (uint16_t phase)
 {
   switch (phase)
   {
     case PHASE_KIND_ALL_TO_ALL: return "ALL_TO_ALL";
+    case PHASE_KIND_ALL_TO_ALL_2: return "ALL_TO_ALL_2";
     case PHASE_KIND_FINISH: return "FINISH";
     case PHASE_KIND_GRADECAST_LEADER: return "GRADECAST_LEADER";
     case PHASE_KIND_GRADECAST_ECHO: return "GRADECAST_ECHO";
@@ -652,36 +657,6 @@ debug_str_rfn_key (const struct RfnKey *rk)
 #endif /* GNUNET_EXTRA_LOGGING */
 
 
-/**
- * Destroy a session, free all resources associated with it.
- *
- * @param session the session to destroy
- */
-static void
-destroy_session (struct ConsensusSession *session)
-{
-  GNUNET_CONTAINER_DLL_remove (sessions_head, sessions_tail, session);
-  if (NULL != session->set_listener)
-  {
-    GNUNET_SET_listen_cancel (session->set_listener);
-    session->set_listener = NULL;
-  }
-  if (NULL != session->client_mq)
-  {
-    GNUNET_MQ_destroy (session->client_mq);
-    session->client_mq = NULL;
-    /* The MQ cleanup will also disconnect the underlying client. */
-    session->client = NULL;
-  }
-  if (NULL != session->client)
-  {
-    GNUNET_SERVER_client_disconnect (session->client);
-    session->client = NULL;
-  }
-  GNUNET_free (session);
-}
-
-
 /**
  * Send the final result set of the consensus to the client, element by
  * element.
@@ -702,16 +677,25 @@ send_to_client_iter (void *cls,
   if (NULL != element)
   {
     struct GNUNET_CONSENSUS_ElementMessage *m;
+    const struct ConsensusElement *ce;
+
+    GNUNET_assert (GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT == element->element_type);
+    ce = element->data;
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "marker is %u\n", (unsigned) ce->marker);
+
+    if (0 != ce->marker)
+      return GNUNET_YES;
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "P%d: sending element %s to client\n",
                 session->local_peer_idx,
                 debug_str_element (element));
 
-    ev = GNUNET_MQ_msg_extra (m, element->size,
+    ev = GNUNET_MQ_msg_extra (m, element->size - sizeof (struct ConsensusElement),
                               GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_RECEIVED_ELEMENT);
-    m->element_type = htons (element->element_type);
-    memcpy (&m[1], element->data, element->size);
+    m->element_type = ce->payload_type;
+    GNUNET_memcpy (&m[1], &ce[1], element->size - sizeof (struct ConsensusElement));
     GNUNET_MQ_send (session->client_mq, ev);
   }
   else
@@ -881,7 +865,7 @@ rfn_vote (struct ReferendumEntry *rfn,
 }
 
 
-uint16_t
+static uint16_t
 task_other_peer (struct TaskEntry *task)
 {
   uint16_t me = task->step->session->local_peer_idx;
@@ -891,17 +875,33 @@ task_other_peer (struct TaskEntry *task)
 }
 
 
+static int
+cmp_uint64_t (const void *pa, const void *pb)
+{
+  uint64_t a = *(uint64_t *) pa;
+  uint64_t b = *(uint64_t *) pb;
+
+  if (a == b)
+    return 0;
+  if (a < b)
+    return -1;
+  return 1;
+}
+
+
 /**
  * Callback for set operation results. Called for each element
  * in the result set.
  *
  * @param cls closure
  * @param element a result element, only valid if status is GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status see enum GNUNET_SET_Status
  */
 static void
 set_result_cb (void *cls,
                const struct GNUNET_SET_Element *element,
+               uint64_t current_size,
                enum GNUNET_SET_Status status)
 {
   struct TaskEntry *task = cls;
@@ -911,6 +911,18 @@ set_result_cb (void *cls,
   struct ReferendumEntry *output_rfn = NULL;
   unsigned int other_idx;
   struct SetOpCls *setop;
+  const struct ConsensusElement *consensus_element = NULL;
+
+  if (NULL != element)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "P%u: got element of type %u, status %u\n",
+                session->local_peer_idx,
+                (unsigned) element->element_type,
+                (unsigned) status);
+    GNUNET_assert (GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT == element->element_type);
+    consensus_element = element->data;
+  }
 
   setop = &task->cls.setop;
 
@@ -963,19 +975,53 @@ set_result_cb (void *cls,
       return;
   }
 
-  if ( (GNUNET_SET_STATUS_ADD_LOCAL == status) || (GNUNET_SET_STATUS_ADD_REMOTE == status) )
+  if ( (NULL != consensus_element) && (0 != consensus_element->marker) )
   {
-    if ( (GNUNET_YES == setop->transceive_contested) && (ELEMENT_TYPE_CONTESTED_MARKER == element->element_type) )
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "P%u: got some marker\n",
+                  session->local_peer_idx);
+    if ( (GNUNET_YES == setop->transceive_contested) &&
+         (CONSENSUS_MARKER_CONTESTED == consensus_element->marker) )
     {
       GNUNET_assert (NULL != output_rfn);
       rfn_contest (output_rfn, task_other_peer (task));
       return;
     }
+
+    if (CONSENSUS_MARKER_SIZE == consensus_element->marker)
+    {
+
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "P%u: got size marker\n",
+                  session->local_peer_idx);
+
+
+      struct ConsensusSizeElement *cse = (void *) consensus_element;
+
+      if (cse->sender_index == other_idx)
+      {
+        if (NULL == session->first_sizes_received)
+          session->first_sizes_received = GNUNET_new_array (session->num_peers, uint64_t);
+        session->first_sizes_received[other_idx] = GNUNET_ntohll (cse->size);
+
+        uint64_t *copy = GNUNET_memdup (session->first_sizes_received, sizeof (uint64_t) * session->num_peers);
+        qsort (copy, session->num_peers, sizeof (uint64_t), cmp_uint64_t);
+        session->lower_bound = copy[session->num_peers / 3 + 1];
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "P%u: lower bound %llu\n",
+                    session->local_peer_idx,
+                    (long long) session->lower_bound);
+      }
+      return;
+    }
+
+    return;
   }
 
   switch (status)
   {
     case GNUNET_SET_STATUS_ADD_LOCAL:
+      GNUNET_assert (NULL != consensus_element);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Adding element in Task {%s}\n",
                   debug_str_task_key (&task->key));
@@ -1022,9 +1068,10 @@ set_result_cb (void *cls,
       // XXX: add result to structures in task
       break;
     case GNUNET_SET_STATUS_ADD_REMOTE:
+      GNUNET_assert (NULL != consensus_element);
       if (GNUNET_YES == setop->do_not_remove)
         break;
-      if (ELEMENT_TYPE_CONTESTED_MARKER == element->element_type)
+      if (CONSENSUS_MARKER_CONTESTED == consensus_element->marker)
         break;
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Removing element in Task {%s}\n",
@@ -1080,6 +1127,10 @@ set_result_cb (void *cls,
       {
         rfn_commit (output_rfn, task_other_peer (task));
       }
+      if (PHASE_KIND_ALL_TO_ALL == task->key.kind)
+      {
+        session->first_size = current_size;
+      }
       finish_task (task);
       break;
     case GNUNET_SET_STATUS_FAILURE:
@@ -1102,6 +1153,7 @@ enum EvilnessType
   EVILNESS_CRAM_LEAD,
   EVILNESS_CRAM_ECHO,
   EVILNESS_SLACK,
+  EVILNESS_SLACK_A2A,
 };
 
 enum EvilnessSubType
@@ -1194,6 +1246,10 @@ get_evilness (struct ConsensusSession *session, struct Evilness *evil)
       {
         evil->type = EVILNESS_SLACK;
       }
+      if (0 == strcmp ("slack-a2a", evil_type_str))
+      {
+        evil->type = EVILNESS_SLACK_A2A;
+      }
       else if (0 == strcmp ("cram-all", evil_type_str))
       {
         evil->type = EVILNESS_CRAM_ALL;
@@ -1259,6 +1315,31 @@ commit_set (struct ConsensusSession *session,
   set = lookup_set (session, &setop->input_set);
   GNUNET_assert (NULL != set);
 
+  if ( (GNUNET_YES == setop->transceive_contested) && (GNUNET_YES == set->is_contested) )
+  {
+    struct GNUNET_SET_Element element;
+    struct ConsensusElement ce = { 0 };
+    ce.marker = CONSENSUS_MARKER_CONTESTED;
+    element.data = &ce;
+    element.size = sizeof (struct ConsensusElement);
+    element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
+    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
+  }
+
+  if (PHASE_KIND_ALL_TO_ALL_2 == task->key.kind)
+  {
+    struct GNUNET_SET_Element element;
+    struct ConsensusSizeElement cse = { 0 };
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "inserting size marker\n");
+    cse.ce.marker = CONSENSUS_MARKER_SIZE;
+    cse.size = GNUNET_htonll (session->first_size);
+    cse.sender_index = session->local_peer_idx;
+    element.data = &cse;
+    element.size = sizeof (struct ConsensusSizeElement);
+    element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
+    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
+  }
+
 #ifdef EVIL
   {
     unsigned int i;
@@ -1300,21 +1381,21 @@ commit_set (struct ConsensusSession *session,
         }
         for (i = 0; i < evil.num; i++)
         {
-          struct GNUNET_HashCode hash;
           struct GNUNET_SET_Element element;
-          element.data = &hash;
-          element.size = sizeof (struct GNUNET_HashCode);
-          element.element_type = 0;
+          struct ConsensusStuffedElement se = { 0 };
+          element.data = &se;
+          element.size = sizeof (struct ConsensusStuffedElement);
+          element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
 
           if (EVILNESS_SUB_REPLACEMENT == evil.subtype)
           {
             /* Always generate a new element. */
-            GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &hash);
+            GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &se.rand);
           }
           else if (EVILNESS_SUB_NO_REPLACEMENT == evil.subtype)
           {
             /* Always cram the same elements, derived from counter. */
-            GNUNET_CRYPTO_hash (&i, sizeof (i), &hash);
+            GNUNET_CRYPTO_hash (&i, sizeof (i), &se.rand);
           }
           else
           {
@@ -1341,6 +1422,19 @@ commit_set (struct ConsensusSession *session,
                     "P%u: evil peer: slacking\n",
                     (unsigned int) session->local_peer_idx);
         /* Do nothing. */
+      case EVILNESS_SLACK_A2A:
+        if ( (PHASE_KIND_ALL_TO_ALL_2 == task->key.kind ) ||
+             (PHASE_KIND_ALL_TO_ALL == task->key.kind) )
+        {
+          struct GNUNET_SET_Handle *empty_set;
+          empty_set = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+          GNUNET_SET_commit (setop->op, empty_set);
+          GNUNET_SET_destroy (empty_set);
+        }
+        else
+        {
+          GNUNET_SET_commit (setop->op, set->h);
+        }
         break;
       case EVILNESS_NONE:
         GNUNET_SET_commit (setop->op, set->h);
@@ -1348,15 +1442,6 @@ commit_set (struct ConsensusSession *session,
     }
   }
 #else
-  if ( (GNUNET_YES == setop->transceive_contested) && (GNUNET_YES == set->is_contested) )
-  {
-    struct GNUNET_SET_Element element;
-    struct ContestedPayload payload;
-    element.data = &payload;
-    element.size = sizeof (struct ContestedPayload);
-    element.element_type = ELEMENT_TYPE_CONTESTED_MARKER;
-    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
-  }
   if (GNUNET_NO == session->peers_blacklisted[task_other_peer (task)])
   {
     GNUNET_SET_commit (setop->op, set->h);
@@ -1511,12 +1596,14 @@ rfn_create (uint16_t size)
 }
 
 
-void
+#if UNUSED
+static void
 diff_destroy (struct DiffEntry *diff)
 {
   GNUNET_CONTAINER_multihashmap_destroy (diff->changes);
   GNUNET_free (diff);
 }
+#endif
 
 
 /**
@@ -1758,7 +1845,10 @@ task_start_apply_round (struct TaskEntry *task)
 
   iter = GNUNET_CONTAINER_multihashmap_iterator_create (rfn_in->rfn_elements);
 
-  while (GNUNET_YES == GNUNET_CONTAINER_multihashmap_iterator_next (iter, NULL, (const void **) &ri))
+  while (GNUNET_YES ==
+         GNUNET_CONTAINER_multihashmap_iterator_next (iter,
+                                                      NULL,
+                                                      (const void **) &ri))
   {
     uint16_t majority_num;
     enum ReferendumVote majority_vote;
@@ -1775,7 +1865,7 @@ task_start_apply_round (struct TaskEntry *task)
         GNUNET_assert (GNUNET_OK ==
                        GNUNET_SET_add_element (set_out->h,
                                                ri->element,
-                                               set_mutation_done,
+                                               &set_mutation_done,
                                                progress_cls));
         GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                     "P%u: apply round: adding element %s with %u-majority.\n",
@@ -1787,7 +1877,7 @@ task_start_apply_round (struct TaskEntry *task)
         GNUNET_assert (GNUNET_OK ==
                        GNUNET_SET_remove_element (set_out->h,
                                                   ri->element,
-                                                  set_mutation_done,
+                                                  &set_mutation_done,
                                                   progress_cls));
         GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                     "P%u: apply round: deleting element %s with %u-majority.\n",
@@ -1807,7 +1897,7 @@ task_start_apply_round (struct TaskEntry *task)
     }
   }
 
-  if (progress_cls->num_pending == 0)
+  if (0 == progress_cls->num_pending)
   {
     // call closure right now, no pending ops
     GNUNET_free (progress_cls);
@@ -1858,7 +1948,7 @@ task_start_apply_round (struct TaskEntry *task)
                   session->local_peer_idx);
     }
   }
-
+  GNUNET_CONTAINER_multihashmap_iterator_destroy (iter);
 }
 
 
@@ -1896,7 +1986,10 @@ task_start_grade (struct TaskEntry *task)
 
   apply_diff_to_rfn (input_diff, output_rfn, task->key.leader, session->num_peers);
 
-  while (GNUNET_YES == GNUNET_CONTAINER_multihashmap_iterator_next (iter, NULL, (const void **) &ri))
+  while (GNUNET_YES ==
+         GNUNET_CONTAINER_multihashmap_iterator_next (iter,
+                                                      NULL,
+                                                      (const void **) &ri))
   {
     uint16_t majority_num;
     enum ReferendumVote majority_vote;
@@ -1922,6 +2015,7 @@ task_start_grade (struct TaskEntry *task)
         break;
     }
   }
+  GNUNET_CONTAINER_multihashmap_iterator_destroy (iter);
 
   {
     uint16_t noncontested;
@@ -2031,12 +2125,18 @@ task_start_reconcile (struct TaskEntry *task)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: initiating set op with P%u, our set is %s\n",
                 session->local_peer_idx, task->key.peer2, debug_str_set_key (&setop->input_set));
 
+    struct GNUNET_SET_Option opts[] = {
+      { GNUNET_SET_OPTION_BYZANTINE, { .num = session->lower_bound } },
+      { GNUNET_SET_OPTION_END },
+    };
+
     // XXX: maybe this should be done while
     // setting up tasks alreays?
     setop->op = GNUNET_SET_prepare (&session->peers[task->key.peer2],
                                     &session->global_id,
                                     &rcm.header,
                                     GNUNET_SET_RESULT_SYMMETRIC,
+                                    opts,
                                     set_result_cb,
                                     task);
 
@@ -2158,7 +2258,7 @@ task_start_eval_echo (struct TaskEntry *task)
 
   GNUNET_CONTAINER_multihashmap_iterator_destroy (iter);
 
-  if (progress_cls->num_pending == 0)
+  if (0 == progress_cls->num_pending)
   {
     // call closure right now, no pending ops
     GNUNET_free (progress_cls);
@@ -2321,45 +2421,48 @@ peer_id_cmp (const void *h1, const void *h2)
 /**
  * Create the sorted list of peers for the session,
  * add the local peer if not in the join message.
+ *
+ * @param session session to initialize
+ * @param join_msg join message with the list of peers participating at the end
  */
 static void
 initialize_session_peer_list (struct ConsensusSession *session,
-                              struct GNUNET_CONSENSUS_JoinMessage *join_msg)
+                              const struct GNUNET_CONSENSUS_JoinMessage *join_msg)
 {
-  unsigned int local_peer_in_list;
-  uint32_t listed_peers;
-  const struct GNUNET_PeerIdentity *msg_peers;
-  unsigned int i;
-
-  GNUNET_assert (NULL != join_msg);
+  const struct GNUNET_PeerIdentity *msg_peers
+    = (const struct GNUNET_PeerIdentity *) &join_msg[1];
+  int local_peer_in_list;
 
-  /* peers in the join message, may or may not include the local peer */
-  listed_peers = ntohl (join_msg->num_peers);
-
-  session->num_peers = listed_peers;
-
-  msg_peers = (struct GNUNET_PeerIdentity *) &join_msg[1];
+  session->num_peers = ntohl (join_msg->num_peers);
 
+  /* Peers in the join message, may or may not include the local peer,
+     Add it if it is missing. */
   local_peer_in_list = GNUNET_NO;
-  for (i = 0; i < listed_peers; i++)
+  for (unsigned int i = 0; i < session->num_peers; i++)
   {
-    if (0 == memcmp (&msg_peers[i], &my_peer, sizeof (struct GNUNET_PeerIdentity)))
+    if (0 == memcmp (&msg_peers[i],
+                     &my_peer,
+                     sizeof (struct GNUNET_PeerIdentity)))
     {
       local_peer_in_list = GNUNET_YES;
       break;
     }
   }
-
   if (GNUNET_NO == local_peer_in_list)
     session->num_peers++;
 
-  session->peers = GNUNET_malloc (session->num_peers * sizeof (struct GNUNET_PeerIdentity));
-
+  session->peers = GNUNET_new_array (session->num_peers,
+                                     struct GNUNET_PeerIdentity);
   if (GNUNET_NO == local_peer_in_list)
     session->peers[session->num_peers - 1] = my_peer;
 
-  memcpy (session->peers, msg_peers, listed_peers * sizeof (struct GNUNET_PeerIdentity));
-  qsort (session->peers, session->num_peers, sizeof (struct GNUNET_PeerIdentity), &peer_id_cmp);
+  GNUNET_memcpy (session->peers,
+                 msg_peers,
+                 ntohl (join_msg->num_peers) * sizeof (struct GNUNET_PeerIdentity));
+  qsort (session->peers,
+         session->num_peers,
+         sizeof (struct GNUNET_PeerIdentity),
+         &peer_id_cmp);
 }
 
 
@@ -2458,8 +2561,14 @@ set_listen_cb (void *cls,
   GNUNET_assert (! ((task->key.peer1 == session->local_peer_idx) &&
                     (task->key.peer2 == session->local_peer_idx)));
 
+  struct GNUNET_SET_Option opts[] = {
+    { GNUNET_SET_OPTION_BYZANTINE, { .num = session->lower_bound } },
+    { GNUNET_SET_OPTION_END },
+  };
+
   task->cls.setop.op = GNUNET_SET_accept (request,
                                           GNUNET_SET_RESULT_SYMMETRIC,
+                                          opts,
                                           set_result_cb,
                                           task);
 
@@ -2855,11 +2964,42 @@ construct_task_graph (struct ConsensusSession *session)
     put_task (session->taskmap, &task);
   }
 
+  round += 1;
   prev_step = step;
-  step = NULL;
+  step = create_step (session, round, GNUNET_NO);;
+#ifdef GNUNET_EXTRA_LOGGING
+  step->debug_name = GNUNET_strdup ("all to all 2");
+#endif
+  step_depend_on (step, prev_step);
+
+
+  for (i = 0; i < n; i++)
+  {
+    uint16_t p1;
+    uint16_t p2;
+
+    p1 = me;
+    p2 = i;
+    arrange_peers (&p1, &p2, n);
+    task = ((struct TaskEntry) {
+      .key = (struct TaskKey) { PHASE_KIND_ALL_TO_ALL_2, p1, p2, -1, -1 },
+      .step = step,
+      .start = task_start_reconcile,
+      .cancel = task_cancel_reconcile,
+    });
+    task.cls.setop.input_set = (struct SetKey) { SET_KIND_CURRENT, 0 };
+    task.cls.setop.output_set = task.cls.setop.input_set;
+    task.cls.setop.do_not_remove = GNUNET_YES;
+    put_task (session->taskmap, &task);
+  }
 
   round += 1;
 
+  prev_step = step;
+  step = NULL;
+
+
+
   /* Byzantine union */
 
   /* sequential repetitions of the gradecasts */
@@ -2917,250 +3057,225 @@ construct_task_graph (struct ConsensusSession *session)
 }
 
 
+
+/**
+ * Check join message.
+ *
+ * @param cls session of client that sent the message
+ * @param m message sent by the client
+ * @return #GNUNET_OK if @a m is well-formed
+ */
+static int
+check_client_join (void *cls,
+                   const struct GNUNET_CONSENSUS_JoinMessage *m)
+{
+  uint32_t listed_peers = ntohl (m->num_peers);
+
+  if ( (ntohs (m->header.size) - sizeof (*m)) !=
+       listed_peers * sizeof (struct GNUNET_PeerIdentity))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
 /**
- * Initialize the session, continue receiving messages from the owning client
+ * Called when a client wants to join a consensus session.
  *
- * @param session the session to initialize
- * @param join_msg the join message from the client
+ * @param cls session of client that sent the message
+ * @param m message sent by the client
  */
 static void
-initialize_session (struct ConsensusSession *session,
-                    struct GNUNET_CONSENSUS_JoinMessage *join_msg)
+handle_client_join (void *cls,
+                    const struct GNUNET_CONSENSUS_JoinMessage *m)
 {
+  struct ConsensusSession *session = cls;
   struct ConsensusSession *other_session;
 
-  initialize_session_peer_list (session, join_msg);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "session with %u peers\n", session->num_peers);
-  compute_global_id (session, &join_msg->session_id);
+  initialize_session_peer_list (session,
+                                m);
+  compute_global_id (session,
+                     &m->session_id);
 
   /* Check if some local client already owns the session.
      It is only legal to have a session with an existing global id
      if all other sessions with this global id are finished.*/
-  other_session = sessions_head;
-  while (NULL != other_session)
+  for (other_session = sessions_head;
+       NULL != other_session;
+       other_session = other_session->next)
   {
-    if ((other_session != session) &&
-        (0 == GNUNET_CRYPTO_hash_cmp (&session->global_id, &other_session->global_id)))
-    {
-      //if (CONSENSUS_ROUND_FINISH != other_session->current_round)
-      //{
-      //  GNUNET_break (0);
-      //  destroy_session (session);
-      //  return;
-      //}
+    if ( (other_session != session) &&
+         (0 == GNUNET_CRYPTO_hash_cmp (&session->global_id,
+                                       &other_session->global_id)) )
       break;
-    }
-    other_session = other_session->next;
   }
 
-  session->conclude_deadline = GNUNET_TIME_absolute_ntoh (join_msg->deadline);
-  session->conclude_start = GNUNET_TIME_absolute_ntoh (join_msg->start);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "consensus with timeout %llums created\n",
-              (long long) (GNUNET_TIME_absolute_get_difference (session->conclude_start, session->conclude_deadline)).rel_value_us / 1000);
-
-  session->local_peer_idx = get_peer_idx (&my_peer, session);
+  session->conclude_deadline
+    = GNUNET_TIME_absolute_ntoh (m->deadline);
+  session->conclude_start
+    = GNUNET_TIME_absolute_ntoh (m->start);
+  session->local_peer_idx = get_peer_idx (&my_peer,
+                                          session);
   GNUNET_assert (-1 != session->local_peer_idx);
-  session->set_listener = GNUNET_SET_listen (cfg, GNUNET_SET_OPERATION_UNION,
-                                             &session->global_id,
-                                             set_listen_cb, session);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d is the local peer\n", session->local_peer_idx);
 
-  session->setmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->taskmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->diffmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->rfnmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Joining consensus session %s containing %u peers as %u with timeout %s\n",
+              GNUNET_h2s (&m->session_id),
+              session->num_peers,
+              session->local_peer_idx,
+              GNUNET_STRINGS_relative_time_to_string
+              (GNUNET_TIME_absolute_get_difference (session->conclude_start,
+                                                    session->conclude_deadline),
+               GNUNET_YES));
+
+  session->set_listener
+    = GNUNET_SET_listen (cfg,
+                         GNUNET_SET_OPERATION_UNION,
+                         &session->global_id,
+                         &set_listen_cb,
+                         session);
+
+  session->setmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                          GNUNET_NO);
+  session->taskmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                           GNUNET_NO);
+  session->diffmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                           GNUNET_NO);
+  session->rfnmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                          GNUNET_NO);
 
   {
     struct SetEntry *client_set;
+
     client_set = GNUNET_new (struct SetEntry);
-    client_set->h = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+    client_set->h = GNUNET_SET_create (cfg,
+                                       GNUNET_SET_OPERATION_UNION);
     client_set->key = ((struct SetKey) { SET_KIND_CURRENT, 0, 0 });
-    put_set (session, client_set);
+    put_set (session,
+             client_set);
   }
 
-  session->peers_blacklisted = GNUNET_new_array (session->num_peers, int);
+  session->peers_blacklisted = GNUNET_new_array (session->num_peers,
+                                                 int);
 
   /* Just construct the task graph,
      but don't run anything until the client calls conclude. */
   construct_task_graph (session);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "session %s initialized\n", GNUNET_h2s (&session->global_id));
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
-static struct ConsensusSession *
-get_session_by_client (struct GNUNET_SERVER_Client *client)
+static void
+client_insert_done (void *cls)
 {
-  struct ConsensusSession *session;
-
-  session = sessions_head;
-  while (NULL != session)
-  {
-    if (session->client == client)
-      return session;
-    session = session->next;
-  }
-  return NULL;
+  // FIXME: implement
 }
 
 
 /**
- * Called when a client wants to join a consensus session.
+ * Called when a client performs an insert operation.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client handle
+ * @param msg message sent by the client
+ * @return #GNUNET_OK (always well-formed)
  */
-static void
-client_join (void *cls,
-             struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *m)
-{
-  struct ConsensusSession *session;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "join message sent by client\n");
-
-  session = get_session_by_client (client);
-  if (NULL != session)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  session = GNUNET_new (struct ConsensusSession);
-  session->client = client;
-  session->client_mq = GNUNET_MQ_queue_for_server_client (client);
-  GNUNET_CONTAINER_DLL_insert (sessions_head, sessions_tail, session);
-  initialize_session (session, (struct GNUNET_CONSENSUS_JoinMessage *) m);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "join done\n");
-}
-
-
-static void
-client_insert_done (void *cls)
+static int
+check_client_insert (void *cls,
+                      const struct GNUNET_CONSENSUS_ElementMessage *msg)
 {
-  // FIXME: implement
+  return GNUNET_OK;
 }
 
 
 /**
  * Called when a client performs an insert operation.
  *
- * @param cls (unused)
- * @param client client handle
- * @param m message sent by the client
+ * @param cls client handle
+ * @param msg message sent by the client
  */
-void
-client_insert (void *cls,
-               struct GNUNET_SERVER_Client *client,
-               const struct GNUNET_MessageHeader *m)
+static void
+handle_client_insert (void *cls,
+                      const struct GNUNET_CONSENSUS_ElementMessage *msg)
 {
-  struct ConsensusSession *session;
-  struct GNUNET_CONSENSUS_ElementMessage *msg;
-  struct GNUNET_SET_Element *element;
+  struct ConsensusSession *session = cls;
   ssize_t element_size;
   struct GNUNET_SET_Handle *initial_set;
-
-  session = get_session_by_client (client);
-
-  if (NULL == session)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
+  struct ConsensusElement *ce;
 
   if (GNUNET_YES == session->conclude_started)
   {
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (session->client);
     return;
   }
 
-  msg = (struct GNUNET_CONSENSUS_ElementMessage *) m;
   element_size = ntohs (msg->header.size) - sizeof (struct GNUNET_CONSENSUS_ElementMessage);
-  if (element_size < 0)
-  {
-    GNUNET_break (0);
-    return;
-  }
+  ce = GNUNET_malloc (sizeof (struct ConsensusElement) + element_size);
+  GNUNET_memcpy (&ce[1], &msg[1], element_size);
+  ce->payload_type = msg->element_type;
+
+  struct GNUNET_SET_Element element = {
+    .element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT,
+    .size = sizeof (struct ConsensusElement) + element_size,
+    .data = ce,
+  };
 
-  element = GNUNET_malloc (sizeof (struct GNUNET_SET_Element) + element_size);
-  element->element_type = msg->element_type;
-  element->size = element_size;
-  memcpy (&element[1], &msg[1], element_size);
-  element->data = &element[1];
   {
     struct SetKey key = { SET_KIND_CURRENT, 0, 0 };
     struct SetEntry *entry;
-    entry = lookup_set (session, &key);
+
+    entry = lookup_set (session,
+                        &key);
     GNUNET_assert (NULL != entry);
     initial_set = entry->h;
   }
+
   session->num_client_insert_pending++;
-  GNUNET_SET_add_element (initial_set, element, client_insert_done, session);
+  GNUNET_SET_add_element (initial_set,
+                          &element,
+                          &client_insert_done,
+                          session);
 
 #ifdef GNUNET_EXTRA_LOGGING
   {
-    struct GNUNET_HashCode hash;
-
-    GNUNET_SET_element_hash (element, &hash);
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: element %s added\n",
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "P%u: element %s added\n",
                 session->local_peer_idx,
-                GNUNET_h2s (&hash));
+                debug_str_element (&element));
   }
 #endif
-
-  GNUNET_free (element);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_free (ce);
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
 /**
  * Called when a client performs the conclude operation.
  *
- * @param cls (unused)
- * @param client client handle
+ * @param cls client handle
  * @param message message sent by the client
  */
 static void
-client_conclude (void *cls,
-                 struct GNUNET_SERVER_Client *client,
-                 const struct GNUNET_MessageHeader *message)
+handle_client_conclude (void *cls,
+                        const struct GNUNET_MessageHeader *message)
 {
-  struct ConsensusSession *session;
-
-  session = get_session_by_client (client);
-  if (NULL == session)
-  {
-    /* client not found */
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
+  struct ConsensusSession *session = cls;
 
   if (GNUNET_YES == session->conclude_started)
   {
     /* conclude started twice */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    destroy_session (session);
+    GNUNET_SERVICE_client_drop (session->client);
     return;
   }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "conclude requested\n");
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "conclude requested\n");
   session->conclude_started = GNUNET_YES;
-
   install_step_timeouts (session);
   run_ready_steps (session);
-
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
@@ -3172,82 +3287,115 @@ client_conclude (void *cls,
 static void
 shutdown_task (void *cls)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "shutting down\n");
-  while (NULL != sessions_head)
-    destroy_session (sessions_head);
-
-  GNUNET_STATISTICS_destroy (statistics, GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "shutting down\n");
+  GNUNET_STATISTICS_destroy (statistics,
+                             GNUNET_NO);
+  statistics = NULL;
 }
 
 
 /**
- * Clean up after a client after it is
- * disconnected (either by us or by itself)
+ * Start processing consensus requests.
  *
- * @param cls closure, unused
- * @param client the client to clean up after
+ * @param cls closure
+ * @param c configuration to use
+ * @param service the initialized service
  */
-void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  struct ConsensusSession *session;
-
-  session = get_session_by_client (client);
-  if (NULL == session)
+  cfg = c;
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_get_peer_identity (cfg,
+                                       &my_peer))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Could not retrieve host identity\n");
+    GNUNET_SCHEDULER_shutdown ();
     return;
-  // FIXME: destroy if we can
+  }
+  statistics = GNUNET_STATISTICS_create ("consensus",
+                                         cfg);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
 }
 
 
+/**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                  struct GNUNET_SERVICE_Client *c,
+                  struct GNUNET_MQ_Handle *mq)
+{
+  struct ConsensusSession *session = GNUNET_new (struct ConsensusSession);
+
+  session->client = c;
+  session->client_mq = mq;
+  GNUNET_CONTAINER_DLL_insert (sessions_head,
+                               sessions_tail,
+                               session);
+  return session;
+}
+
 
 /**
- * Start processing consensus requests.
+ * Callback called when a client disconnected from the service
  *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  static const struct GNUNET_SERVER_MessageHandler server_handlers[] = {
-    {&client_conclude, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_CONCLUDE,
-        sizeof (struct GNUNET_MessageHeader)},
-    {&client_insert, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_INSERT, 0},
-    {&client_join, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_JOIN, 0},
-    {NULL, NULL, 0, 0}
-  };
+client_disconnect_cb (void *cls,
+                     struct GNUNET_SERVICE_Client *c,
+                     void *internal_cls)
+{
+  struct ConsensusSession *session = internal_cls;
 
-  cfg = c;
-  srv = server;
-  if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &my_peer))
+  if (NULL != session->set_listener)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+    GNUNET_SET_listen_cancel (session->set_listener);
+    session->set_listener = NULL;
   }
-  statistics = GNUNET_STATISTICS_create ("consensus", cfg);
-  GNUNET_SERVER_add_handlers (server, server_handlers);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-  GNUNET_SERVER_disconnect_notify (server, handle_client_disconnect, NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "consensus running\n");
+  GNUNET_CONTAINER_DLL_remove (sessions_head,
+                               sessions_tail,
+                               session);
+  GNUNET_free (session);
 }
 
 
 /**
- * The main function for the consensus service.
- *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * Define "main" method using service macro.
  */
-int
-main (int argc, char *const *argv)
-{
-  int ret;
-  ret = GNUNET_SERVICE_run (argc, argv, "consensus", GNUNET_SERVICE_OPTION_NONE, &run, NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "exit (%d)\n", GNUNET_OK != ret);
-  return (GNUNET_OK == ret) ? 0 : 1;
-}
+GNUNET_SERVICE_MAIN
+("consensus",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (client_conclude,
+                          GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_CONCLUDE,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_var_size (client_insert,
+                        GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_INSERT,
+                        struct GNUNET_CONSENSUS_ElementMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (client_join,
+                        GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_JOIN,
+                        struct GNUNET_CONSENSUS_JoinMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
+/* end of gnunet-service-consensus.c */