log: add \n
[oweals/gnunet.git] / src / set / gnunet-service-set.c
index b2ad01d1b7b8ee9df9998c62507796e04ad4a135..e4e2535af94b84004865a34b4312fe335bb12b04 100644 (file)
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 Christian Grothoff (and other contributing authors)
+      Copyright (C) 2013, 2014 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -25,6 +25,7 @@
  */
 #include "gnunet-service-set.h"
 #include "gnunet-service-set_protocol.h"
+#include "gnunet_statistics_service.h"
 
 /**
  * How long do we hold on to an incoming channel if there is
@@ -65,6 +66,11 @@ struct Listener
    */
   struct GNUNET_HashCode app_id;
 
+  /**
+   * The port we are listening on with CADET.
+   */
+  struct GNUNET_CADET_Port *open_port;
+
   /**
    * The type of the operation.
    */
@@ -137,6 +143,11 @@ static uint32_t lazy_copy_cookie = 1;
  */
 static uint32_t suggest_id = 1;
 
+/**
+ * Statistics handle.
+ */
+struct GNUNET_STATISTICS_Handle *_GSS_statistics;
+
 
 /**
  * Get set that is owned by the given client, if any.
@@ -223,6 +234,7 @@ listener_destroy (struct Listener *listener)
     GNUNET_MQ_destroy (listener->client_mq);
     listener->client_mq = NULL;
   }
+  GNUNET_CADET_close_port (listener->open_port);
   GNUNET_CONTAINER_DLL_remove (listeners_head,
                                listeners_tail,
                                listener);
@@ -314,7 +326,8 @@ collect_generation_garbage (struct Set *set)
                                          &gc);
 }
 
-int
+
+static int
 is_excluded_generation (unsigned int generation,
                         struct GenerationRange *excluded,
                         unsigned int excluded_size)
@@ -331,7 +344,7 @@ is_excluded_generation (unsigned int generation,
 }
 
 
-int
+static int
 is_element_of_generation (struct ElementEntry *ee,
                           unsigned int query_generation,
                           struct GenerationRange *excluded,
@@ -339,9 +352,9 @@ is_element_of_generation (struct ElementEntry *ee,
 {
   struct MutationEvent *mut;
   int is_present;
+  unsigned int i;
 
-  if (NULL == ee->mutations)
-    return GNUNET_YES;
+  GNUNET_assert (NULL != ee->mutations);
 
   if (GNUNET_YES == is_excluded_generation (query_generation, excluded, excluded_size))
   {
@@ -349,26 +362,42 @@ is_element_of_generation (struct ElementEntry *ee,
     return GNUNET_NO;
   }
 
-  is_present = GNUNET_YES;
+  is_present = GNUNET_NO;
 
-  // Could be made faster with binary search, but lists
-  // are small, so why bother.
-  for (mut = ee->mutations; 0 != mut->generation; mut++)
+  /* Could be made faster with binary search, but lists
+     are small, so why bother. */
+  for (i = 0; i < ee->mutations_size; i++)
   {
-    if ( (mut->generation > query_generation) ||
-         (GNUNET_YES == is_excluded_generation (mut->generation, excluded, excluded_size)) )
+    mut = &ee->mutations[i];
+
+    if (mut->generation > query_generation)
+    {
+      /* The mutation doesn't apply to our generation
+         anymore.  We can'b break here, since mutations aren't
+         sorted by generation. */
+      continue;
+    }
+
+    if (GNUNET_YES == is_excluded_generation (mut->generation, excluded, excluded_size))
     {
+      /* The generation is excluded (because it belongs to another
+         fork via a lazy copy) and thus mutations aren't considered
+         for membership testing. */
       continue;
     }
 
-    // This would be an inconsistency in how we manage mutations.
+    /* This would be an inconsistency in how we manage mutations. */
     if ( (GNUNET_YES == is_present) && (GNUNET_YES == mut->added) )
       GNUNET_assert (0);
 
+    /* Likewise. */
+    if ( (GNUNET_NO == is_present) && (GNUNET_NO == mut->added) )
+      GNUNET_assert (0);
+
     is_present = mut->added;
   }
 
-  return GNUNET_YES;
+  return is_present;
 }
 
 
@@ -383,6 +412,17 @@ _GSS_is_element_of_set (struct ElementEntry *ee,
 }
 
 
+static int
+is_element_of_iteration (struct ElementEntry *ee,
+                         struct Set *set)
+{
+  return is_element_of_generation (ee,
+                                   set->iter_generation,
+                                   set->excluded_generations,
+                                   set->excluded_generations_size);
+}
+
+
 int
 _GSS_is_element_of_operation (struct ElementEntry *ee,
                               struct Operation *op)
@@ -510,8 +550,24 @@ set_destroy (struct Set *set)
   }
   {
     struct SetContent *content;
+    struct PendingMutation *pm;
+    struct PendingMutation *pm_current;
 
     content = set->content;
+
+    // discard any pending mutations that reference this set
+    pm = content->pending_mutations_head;
+    while (NULL != pm)
+    {
+      pm_current = pm;
+      pm = pm->next;
+      if (pm_current-> set == set)
+        GNUNET_CONTAINER_DLL_remove (content->pending_mutations_head,
+                                     content->pending_mutations_tail,
+                                     pm_current);
+
+    }
+
     set->content = NULL;
     GNUNET_assert (0 != content->refcount);
     content->refcount -= 1;
@@ -523,6 +579,7 @@ set_destroy (struct Set *set)
                                              NULL);
       GNUNET_CONTAINER_multihashmap_destroy (content->elements);
       content->elements = NULL;
+      GNUNET_free (content);
     }
   }
   GNUNET_free_non_null (set->excluded_generations);
@@ -531,7 +588,21 @@ set_destroy (struct Set *set)
                                sets_tail,
                                set);
 
-  // FIXME: remove from lazy copy requests
+  // remove set from pending copy requests
+  {
+    struct LazyCopyRequest *lcr;
+    lcr = lazy_copy_head;
+    while (NULL != lcr)
+    {
+      struct LazyCopyRequest *lcr_current;
+      lcr_current = lcr;
+      lcr = lcr->next;
+      if (lcr_current->source_set == set)
+        GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
+                                     lazy_copy_tail,
+                                     lcr_current);
+    }
+  }
 
   GNUNET_free (set);
 }
@@ -547,8 +618,8 @@ static void
 handle_client_disconnect (void *cls,
                           struct GNUNET_SERVER_Client *client)
 {
-  struct Set *set;
   struct Listener *listener;
+  struct Set *set;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "client disconnected, cleaning up\n");
@@ -610,29 +681,6 @@ incoming_destroy (struct Operation *incoming)
 }
 
 
-/**
- * Find a listener that is interested in the given operation type
- * and application id.
- *
- * @param op operation type to look for
- * @param app_id application id to look for
- * @return a matching listener, or NULL if no listener matches the
- *         given operation and application id
- */
-static struct Listener *
-listener_get_by_target (enum GNUNET_SET_OperationType op,
-                        const struct GNUNET_HashCode *app_id)
-{
-  struct Listener *listener;
-
-  for (listener = listeners_head; NULL != listener; listener = listener->next)
-    if ( (listener->operation == op) &&
-         (0 == GNUNET_CRYPTO_hash_cmp (app_id, &listener->app_id)) )
-      return listener;
-  return NULL;
-}
-
-
 /**
  * Suggest the given request to the listener. The listening client can
  * then accept or reject the remote request.
@@ -665,7 +713,8 @@ incoming_suggest (struct Operation *incoming,
               incoming->suggest_id);
   cmsg->accept_id = htonl (incoming->suggest_id);
   cmsg->peer_id = incoming->spec->peer;
-  GNUNET_MQ_send (listener->client_mq, mqm);
+  GNUNET_MQ_send (listener->client_mq,
+                  mqm);
 }
 
 
@@ -691,7 +740,7 @@ handle_incoming_msg (struct Operation *op,
                      const struct GNUNET_MessageHeader *mh)
 {
   const struct OperationRequestMessage *msg;
-  struct Listener *listener;
+  struct Listener *listener = op->listener;
   struct OperationSpecification *spec;
   const struct GNUNET_MessageHeader *nested_context;
 
@@ -723,33 +772,148 @@ handle_incoming_msg (struct Operation *op,
   if (NULL != nested_context)
     spec->context_msg = GNUNET_copy_message (nested_context);
   spec->operation = ntohl (msg->operation);
-  spec->app_id = msg->app_id;
+  spec->app_id = listener->app_id;
   spec->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                          UINT32_MAX);
   spec->peer = op->peer;
   spec->remote_element_count = ntohl (msg->element_count);
   op->spec = spec;
 
-  listener = listener_get_by_target (ntohl (msg->operation),
-                                     &msg->app_id);
-  if (NULL == listener)
-  {
-    GNUNET_break (NULL != op->timeout_task);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "No matching listener for incoming request (op %u, app %s), waiting with timeout\n",
-                ntohl (msg->operation),
-                GNUNET_h2s (&msg->app_id));
-    return GNUNET_OK;
-  }
+  listener = op->listener;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received P2P operation request (op %u, app %s) for active listener\n",
+              "Received P2P operation request (op %u, port %s) for active listener\n",
               ntohl (msg->operation),
-              GNUNET_h2s (&msg->app_id));
-  incoming_suggest (op, listener);
+              GNUNET_h2s (&listener->app_id));
+  incoming_suggest (op,
+                    listener);
   return GNUNET_OK;
 }
 
 
+static void
+execute_add (struct Set *set,
+             const struct GNUNET_MessageHeader *m)
+{
+  const struct GNUNET_SET_ElementMessage *msg;
+  struct GNUNET_SET_Element el;
+  struct ElementEntry *ee;
+  struct GNUNET_HashCode hash;
+
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (m->type));
+
+  msg = (const struct GNUNET_SET_ElementMessage *) m;
+  el.size = ntohs (m->size) - sizeof *msg;
+  el.data = &msg[1];
+  el.element_type = ntohs (msg->element_type);
+  GNUNET_SET_element_hash (&el, &hash);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Client inserts element %s of size %u\n",
+              GNUNET_h2s (&hash),
+              el.size);
+
+  ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
+                                          &hash);
+
+  if (NULL == ee)
+  {
+    ee = GNUNET_malloc (el.size + sizeof *ee);
+    ee->element.size = el.size;
+    GNUNET_memcpy (&ee[1],
+            el.data,
+            el.size);
+    ee->element.data = &ee[1];
+    ee->element.element_type = el.element_type;
+    ee->remote = GNUNET_NO;
+    ee->mutations = NULL;
+    ee->mutations_size = 0;
+    ee->element_hash = hash;
+    GNUNET_break (GNUNET_YES ==
+                  GNUNET_CONTAINER_multihashmap_put (set->content->elements,
+                                                     &ee->element_hash,
+                                                     ee,
+                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  }
+  else if (GNUNET_YES == _GSS_is_element_of_set (ee, set))
+  {
+    /* same element inserted twice */
+    return;
+  }
+
+  {
+    struct MutationEvent mut = {
+      .generation = set->current_generation,
+      .added = GNUNET_YES
+    };
+    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
+  }
+
+  set->vt->add (set->state, ee);
+}
+
+
+static void
+execute_remove (struct Set *set,
+                const struct GNUNET_MessageHeader *m)
+{
+  const struct GNUNET_SET_ElementMessage *msg;
+  struct GNUNET_SET_Element el;
+  struct ElementEntry *ee;
+  struct GNUNET_HashCode hash;
+
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (m->type));
+
+  msg = (const struct GNUNET_SET_ElementMessage *) m;
+  el.size = ntohs (m->size) - sizeof *msg;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Client removes element of size %u\n",
+              el.size);
+  el.data = &msg[1];
+  el.element_type = ntohs (msg->element_type);
+  GNUNET_SET_element_hash (&el, &hash);
+  ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
+                                          &hash);
+  if (NULL == ee)
+  {
+    /* Client tried to remove non-existing element. */
+    return;
+  }
+  if (GNUNET_NO == _GSS_is_element_of_set (ee, set))
+  {
+    /* Client tried to remove element twice */
+    return;
+  }
+  else
+  {
+    struct MutationEvent mut = {
+      .generation = set->current_generation,
+      .added = GNUNET_NO
+    };
+    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
+  }
+  set->vt->remove (set->state, ee);
+}
+
+
+
+static void
+execute_mutation (struct Set *set,
+                  const struct GNUNET_MessageHeader *m)
+{
+  switch (ntohs (m->type))
+  {
+    case GNUNET_MESSAGE_TYPE_SET_ADD:
+      execute_add (set, m);
+      break;
+    case GNUNET_MESSAGE_TYPE_SET_REMOVE:
+      execute_remove (set, m);
+      break;
+    default:
+      GNUNET_break (0);
+  }
+}
+
+
+
 /**
  * Send the next element of a set to the set's client.  The next element is given by
  * the set's current hashmap iterator.  The set's iterator will be set to NULL if there
@@ -772,26 +936,62 @@ send_client_element (struct Set *set)
   struct GNUNET_SET_IterResponseMessage *msg;
 
   GNUNET_assert (NULL != set->iter);
+
+again:
+
   ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
                                                      NULL,
                                                      (const void **) &ee);
   if (GNUNET_NO == ret)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Iteration on %p done.\n",
+                (void *) set);
     ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
     GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
     set->iter = NULL;
     set->iteration_id++;
+
+    GNUNET_assert (set->content->iterator_count > 0);
+    set->content->iterator_count -= 1;
+
+    if (0 == set->content->iterator_count)
+    {
+      while (NULL != set->content->pending_mutations_head)
+      {
+        struct PendingMutation *pm;
+
+        pm = set->content->pending_mutations_head;
+        GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
+                                     set->content->pending_mutations_tail,
+                                     pm);
+        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                    "Executing pending mutation on %p.\n",
+                    (void *) pm->set);
+        execute_mutation (pm->set, pm->mutation_message);
+        GNUNET_free (pm->mutation_message);
+        GNUNET_free (pm);
+      }
+    }
+
   }
   else
   {
     GNUNET_assert (NULL != ee);
+
+    if (GNUNET_NO == is_element_of_iteration (ee, set))
+      goto again;
+
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Sending iteration element on %p.\n",
+                (void *) set);
     ev = GNUNET_MQ_msg_extra (msg,
                               ee->element.size,
                               GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
-    memcpy (&msg[1],
+    GNUNET_memcpy (&msg[1],
             ee->element.data,
             ee->element.size);
-    msg->element_type = ee->element.element_type;
+    msg->element_type = htons (ee->element.element_type);
     msg->iteration_id = htons (set->iteration_id);
   }
   GNUNET_MQ_send (set->client_mq, ev);
@@ -831,11 +1031,15 @@ handle_client_iterate (void *cls,
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Iterating set with %u elements\n",
+              "Iterating set %p in gen %u with %u content elements\n",
+              (void *) set,
+              set->current_generation,
               GNUNET_CONTAINER_multihashmap_size (set->content->elements));
   GNUNET_SERVER_receive_done (client,
                               GNUNET_OK);
+  set->content->iterator_count += 1;
   set->iter = GNUNET_CONTAINER_multihashmap_iterator_create (set->content->elements);
+  set->iter_generation = set->current_generation;
   send_client_element (set);
 }
 
@@ -885,6 +1089,13 @@ handle_client_create_set (void *cls,
   }
   set->operation = ntohl (msg->operation);
   set->state = set->vt->create ();
+  if (NULL == set->state)
+  {
+    /* initialization failed (i.e. out of memory) */
+    GNUNET_free (set);
+    GNUNET_SERVER_client_disconnect (client);
+    return;
+  }
   set->content = GNUNET_new (struct SetContent);
   set->content->refcount = 1;
   set->content->elements = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_YES);
@@ -898,6 +1109,100 @@ handle_client_create_set (void *cls,
 }
 
 
+/**
+ * Timeout happens iff:
+ *  - we suggested an operation to our listener,
+ *    but did not receive a response in time
+ *  - we got the channel from a peer but no #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
+ *
+ * @param cls channel context
+ * @param tc context information (why was this task triggered now)
+ */
+static void
+incoming_timeout_cb (void *cls)
+{
+  struct Operation *incoming = cls;
+
+  incoming->timeout_task = NULL;
+  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Remote peer's incoming request timed out\n");
+  incoming_destroy (incoming);
+}
+
+
+/**
+ * Terminates an incoming operation in case we have not yet received an
+ * operation request. Called by the channel destruction handler.
+ *
+ * @param op the channel context
+ */
+static void
+handle_incoming_disconnect (struct Operation *op)
+{
+  GNUNET_assert (GNUNET_YES == op->is_incoming);
+  /* channel is already dead, incoming_destroy must not
+   * destroy it ... */
+  op->channel = NULL;
+  incoming_destroy (op);
+  op->vt = NULL;
+}
+
+
+/**
+ * Method called whenever another peer has added us to a channel the
+ * other peer initiated.  Only called (once) upon reception of data
+ * from a channel we listen on.
+ *
+ * The channel context represents the operation itself and gets added
+ * to a DLL, from where it gets looked up when our local listener
+ * client responds to a proposed/suggested operation or connects and
+ * associates with this operation.
+ *
+ * @param cls closure
+ * @param channel new handle to the channel
+ * @param initiator peer that started the channel
+ * @param port Port this channel is for.
+ * @param options Unused.
+ * @return initial channel context for the channel
+ *         returns NULL on error
+ */
+static void *
+channel_new_cb (void *cls,
+                struct GNUNET_CADET_Channel *channel,
+                const struct GNUNET_PeerIdentity *initiator,
+                const struct GNUNET_HashCode *port,
+                enum GNUNET_CADET_ChannelOption options)
+{
+  static const struct SetVT incoming_vt = {
+    .msg_handler = &handle_incoming_msg,
+    .peer_disconnect = &handle_incoming_disconnect
+  };
+  struct Listener *listener = cls;
+  struct Operation *incoming;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "New incoming channel\n");
+  incoming = GNUNET_new (struct Operation);
+  incoming->listener = listener;
+  incoming->is_incoming = GNUNET_YES;
+  incoming->peer = *initiator;
+  incoming->channel = channel;
+  incoming->mq = GNUNET_CADET_mq_create (incoming->channel);
+  incoming->vt = &incoming_vt;
+  incoming->timeout_task
+    = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
+                                    &incoming_timeout_cb,
+                                    incoming);
+  GNUNET_CONTAINER_DLL_insert_tail (incoming_head,
+                                    incoming_tail,
+                                    incoming);
+  // incoming_suggest (incoming,
+  //                  listener);
+  return incoming;
+}
+
+
 /**
  * Called when a client wants to create a new listener.
  *
@@ -930,8 +1235,12 @@ handle_client_listen (void *cls,
   GNUNET_CONTAINER_DLL_insert_tail (listeners_head,
                                     listeners_tail,
                                     listener);
+  listener->open_port = GNUNET_CADET_open_port (cadet,
+                                                &msg->app_id,
+                                                &channel_new_cb,
+                                                listener);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "New listener created (op %u, app %s)\n",
+              "New listener created (op %u, port %s)\n",
               listener->operation,
               GNUNET_h2s (&listener->app_id));
 
@@ -952,7 +1261,8 @@ handle_client_listen (void *cls,
     incoming_suggest (op,
                       listener);
   }
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVER_receive_done (client,
+                              GNUNET_OK);
 }
 
 
@@ -992,23 +1302,20 @@ handle_client_reject (void *cls,
 }
 
 
+
 /**
- * Called when a client wants to add an element to a set it inhabits.
+ * Called when a client wants to add or remove an element to a set it inhabits.
  *
  * @param cls unused
  * @param client client that sent the message
  * @param m message sent by the client
  */
 static void
-handle_client_add (void *cls,
-                   struct GNUNET_SERVER_Client *client,
-                   const struct GNUNET_MessageHeader *m)
+handle_client_mutation (void *cls,
+                        struct GNUNET_SERVER_Client *client,
+                        const struct GNUNET_MessageHeader *m)
 {
   struct Set *set;
-  const struct GNUNET_SET_ElementMessage *msg;
-  struct GNUNET_SET_Element el;
-  struct ElementEntry *ee;
-  struct GNUNET_HashCode hash;
 
   set = set_get (client);
   if (NULL == set)
@@ -1018,128 +1325,28 @@ handle_client_add (void *cls,
     GNUNET_SERVER_client_disconnect (client);
     return;
   }
+
   GNUNET_SERVER_receive_done (client,
                               GNUNET_OK);
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Client inserts element of size %u\n",
-              el.size);
-  el.data = &msg[1];
-  GNUNET_CRYPTO_hash (el.data,
-                      el.size,
-                      &hash);
 
-  ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
-                                          &hash);
-
-  if (NULL == ee)
+  if (0 != set->content->iterator_count)
   {
-    ee = GNUNET_malloc (el.size + sizeof *ee);
-    ee->element.size = el.size;
-    memcpy (&ee[1],
-            el.data,
-            el.size);
-    ee->element.data = &ee[1];
-    ee->remote = GNUNET_NO;
-    ee->mutations = NULL;
-    ee->mutations_size = 0;
-    ee->element_hash = hash;
-  } else if (GNUNET_YES == _GSS_is_element_of_set (ee, set)) {
-    /* same element inserted twice */
-    GNUNET_break (0);
-    return;
-  }
-
-  if (0 != set->current_generation)
-  {
-    struct MutationEvent mut = {
-      .generation = set->current_generation,
-      .added = GNUNET_YES
-    };
-    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
-    ee->mutations_size += 1;
-  }
+    struct PendingMutation *pm;
 
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_put (set->content->elements,
-                                                   &ee->element_hash,
-                                                   ee,
-                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  set->vt->add (set->state, ee);
-}
-
-
-/**
- * Called when a client wants to remove an element from a set it
- * inhabits.
- *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
- */
-static void
-handle_client_remove (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *m)
-{
-  struct Set *set;
-  const struct GNUNET_SET_ElementMessage *msg;
-  struct GNUNET_SET_Element el;
-  struct ElementEntry *ee;
-  struct GNUNET_HashCode hash;
-
-  set = set_get (client);
-  if (NULL == set)
-  {
-    /* client without a set requested an operation */
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Client removes element of size %u\n",
-              el.size);
-  el.data = &msg[1];
-  GNUNET_CRYPTO_hash (el.data,
-                      el.size,
-                      &hash);
-  ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
-                                          &hash);
-  if (NULL == ee)
-  {
-    /* Client tried to remove non-existing element */
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_NO == _GSS_is_element_of_set (ee, set))
-  {
-    /* Client tried to remove element twice */
-    GNUNET_break (0);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Scheduling mutation on set\n");
+
+    pm = GNUNET_new (struct PendingMutation);
+    pm->mutation_message = GNUNET_copy_message (m);
+    pm->set = set;
+    GNUNET_CONTAINER_DLL_insert (set->content->pending_mutations_head,
+                                 set->content->pending_mutations_tail,
+                                 pm);
     return;
   }
-  else if (0 == set->current_generation)
-  {
-    // If current_generation is 0, then there are no running set operations
-    // or lazy copies, thus we can safely remove the element.
-    (void) GNUNET_CONTAINER_multihashmap_remove_all (set->content->elements, &hash);
-  }
-  else
-  {
-    struct MutationEvent mut = {
-      .generation = set->current_generation,
-      .added = GNUNET_NO
-    };
-    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
-    ee->mutations_size += 1;
-  }
-  set->vt->remove (set->state, ee);
-}
 
+  execute_mutation (set, m);
+}
 
 
 /**
@@ -1171,10 +1378,9 @@ advance_generation (struct Set *set)
   GNUNET_array_append (set->excluded_generations,
                        set->excluded_generations_size,
                        r);
-
-  set->excluded_generations_size += 1;
 }
 
+
 /**
  * Called when a client wants to initiate a set operation with another
  * peer.  Initiates the CADET connection to the listener and sends the
@@ -1225,10 +1431,13 @@ handle_client_evaluate (void *cls,
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                op);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating new CADET channel to port %s\n",
+              GNUNET_h2s (&msg->app_id));
   op->channel = GNUNET_CADET_channel_create (cadet,
                                              op,
                                              &msg->target_peer,
-                                             GNUNET_APPLICATION_TYPE_SET,
+                                             &msg->app_id,
                                              GNUNET_CADET_OPTION_RELIABLE);
   op->mq = GNUNET_CADET_mq_create (op->channel);
   set->vt->evaluate (op,
@@ -1302,6 +1511,8 @@ handle_client_copy_lazy_prepare (void *cls,
 {
   struct Set *set;
   struct LazyCopyRequest *cr;
+  struct GNUNET_MQ_Envelope *ev;
+  struct GNUNET_SET_CopyLazyResponseMessage *resp_msg;
 
   set = set_get (client);
   if (NULL == set)
@@ -1321,8 +1532,19 @@ handle_client_copy_lazy_prepare (void *cls,
   GNUNET_CONTAINER_DLL_insert (lazy_copy_head,
                                lazy_copy_tail,
                                cr);
+
+
+  ev = GNUNET_MQ_msg (resp_msg,
+                      GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE);
+  resp_msg->cookie = cr->cookie;
+  GNUNET_MQ_send (set->client_mq, ev);
+
+
   GNUNET_SERVER_receive_done (client,
                               GNUNET_OK);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client requested lazy copy\n");
 }
 
 
@@ -1361,7 +1583,7 @@ handle_client_copy_lazy_connect (void *cls,
     {
       found = GNUNET_YES;
       break;
-    } 
+    }
   }
 
   if (GNUNET_NO == found)
@@ -1391,18 +1613,31 @@ handle_client_copy_lazy_connect (void *cls,
     return;
   }
 
-  if (NULL == set->vt->copy_state) {
+  if (NULL == set->vt->copy_state)
+  {
     /* Lazy copy not supported for this set operation */
     GNUNET_break (0);
     GNUNET_free (set);
     GNUNET_free (cr);
+    GNUNET_SERVER_client_disconnect (client);
     return;
   }
 
   set->operation = cr->source_set->operation;
-  set->state = set->vt->copy_state (set);
+  set->state = set->vt->copy_state (cr->source_set);
   set->content = cr->source_set->content;
   set->content->refcount += 1;
+
+  set->current_generation = cr->source_set->current_generation;
+  set->excluded_generations_size = cr->source_set->excluded_generations_size;
+  set->excluded_generations = GNUNET_memdup (cr->source_set->excluded_generations,
+                                             set->excluded_generations_size * sizeof (struct GenerationRange));
+
+  /* Advance the generation of the new set, so that mutations to the
+     of the cloned set and the source set are independent. */
+  advance_generation (set);
+
+
   set->client = client;
   set->client_mq = GNUNET_MQ_queue_for_server_client (client);
   GNUNET_CONTAINER_DLL_insert (sets_head,
@@ -1413,6 +1648,9 @@ handle_client_copy_lazy_connect (void *cls,
 
   GNUNET_SERVER_receive_done (client,
                               GNUNET_OK);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client connected to lazy set\n");
 }
 
 
@@ -1552,11 +1790,9 @@ handle_client_accept (void *cls,
  * Called to clean up, after a shutdown has been requested.
  *
  * @param cls closure
- * @param tc context information (why was this task triggered now)
  */
 static void
-shutdown_task (void *cls,
-               const struct GNUNET_SCHEDULER_TaskContext *tc)
+shutdown_task (void *cls)
 {
   while (NULL != incoming_head)
     incoming_destroy (incoming_head);
@@ -1572,118 +1808,19 @@ shutdown_task (void *cls,
     GNUNET_CADET_disconnect (cadet);
     cadet = NULL;
   }
+  GNUNET_STATISTICS_destroy (_GSS_statistics, GNUNET_YES);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "handled shutdown request\n");
 }
 
 
-/**
- * Timeout happens iff:
- *  - we suggested an operation to our listener,
- *    but did not receive a response in time
- *  - we got the channel from a peer but no #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
- *  - shutdown (obviously)
- *
- * @param cls channel context
- * @param tc context information (why was this task triggered now)
- */
-static void
-incoming_timeout_cb (void *cls,
-                     const struct GNUNET_SCHEDULER_TaskContext *tc)
-{
-  struct Operation *incoming = cls;
-
-  incoming->timeout_task = NULL;
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
-  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
-    return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Remote peer's incoming request timed out\n");
-  incoming_destroy (incoming);
-}
-
-
-/**
- * Terminates an incoming operation in case we have not yet received an
- * operation request. Called by the channel destruction handler.
- *
- * @param op the channel context
- */
-static void
-handle_incoming_disconnect (struct Operation *op)
-{
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  /* channel is already dead, incoming_destroy must not
-   * destroy it ... */
-  op->channel = NULL;
-  incoming_destroy (op);
-  op->vt = NULL;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel the
- * other peer initiated.  Only called (once) upon reception of data
- * with a message type which was subscribed to in
- * GNUNET_CADET_connect().
- *
- * The channel context represents the operation itself and gets added to a DLL,
- * from where it gets looked up when our local listener client responds
- * to a proposed/suggested operation or connects and associates with this operation.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port Port this channel is for.
- * @param options Unused.
- * @return initial channel context for the channel
- *         returns NULL on error
- */
-static void *
-channel_new_cb (void *cls,
-                struct GNUNET_CADET_Channel *channel,
-                const struct GNUNET_PeerIdentity *initiator,
-                uint32_t port,
-                enum GNUNET_CADET_ChannelOption options)
-{
-  static const struct SetVT incoming_vt = {
-    .msg_handler = &handle_incoming_msg,
-    .peer_disconnect = &handle_incoming_disconnect
-  };
-  struct Operation *incoming;
-
-  if (GNUNET_APPLICATION_TYPE_SET != port)
-  {
-    GNUNET_break (0);
-    GNUNET_CADET_channel_destroy (channel);
-    return NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "New incoming channel\n");
-  incoming = GNUNET_new (struct Operation);
-  incoming->is_incoming = GNUNET_YES;
-  incoming->peer = *initiator;
-  incoming->channel = channel;
-  incoming->mq = GNUNET_CADET_mq_create (incoming->channel);
-  incoming->vt = &incoming_vt;
-  incoming->timeout_task
-    = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
-                                    &incoming_timeout_cb,
-                                    incoming);
-  GNUNET_CONTAINER_DLL_insert_tail (incoming_head,
-                                    incoming_tail,
-                                    incoming);
-  return incoming;
-}
-
-
 /**
  * Function called whenever a channel is destroyed.  Should clean up
  * any associated state.  It must NOT call
  * GNUNET_CADET_channel_destroy() on the channel.
  *
  * The peer_disconnect function is part of a a virtual table set initially either
- * when a peer creates a new channel with us (#channel_new_cb()), or once we create
+ * when a peer creates a new channel with us, or once we create
  * a new channel ourselves (evaluate).
  *
  * Once we know the exact type of operation (union/intersection), the vt is
@@ -1728,7 +1865,7 @@ channel_end_cb (void *cls,
  * received via a cadet channel.
  *
  * The msg_handler is a virtual table set in initially either when a peer
- * creates a new channel with us (channel_new_cb), or once we create a new channel
+ * creates a new channel with us, or once we create a new channel
  * ourselves (evaluate).
  *
  * Once we know the exact type of operation (union/intersection), the vt is
@@ -1787,7 +1924,7 @@ run (void *cls,
     { &handle_client_iter_ack, NULL,
       GNUNET_MESSAGE_TYPE_SET_ITER_ACK,
       sizeof (struct GNUNET_SET_IterAckMessage) },
-    { &handle_client_add, NULL,
+    { &handle_client_mutation, NULL,
       GNUNET_MESSAGE_TYPE_SET_ADD,
       0},
     { &handle_client_create_set, NULL,
@@ -1805,7 +1942,7 @@ run (void *cls,
     { &handle_client_reject, NULL,
       GNUNET_MESSAGE_TYPE_SET_REJECT,
       sizeof (struct GNUNET_SET_RejectMessage)},
-    { &handle_client_remove, NULL,
+    { &handle_client_mutation, NULL,
       GNUNET_MESSAGE_TYPE_SET_REMOVE,
       0},
     { &handle_client_cancel, NULL,
@@ -1823,28 +1960,31 @@ run (void *cls,
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE, 0},
+    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER, 0},
+    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY, 0},
+    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS, 0},
+    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE, 0},
+    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF, 0},
     { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE, 0},
     {NULL, 0, 0}
   };
-  static const uint32_t cadet_ports[] = {GNUNET_APPLICATION_TYPE_SET, 0};
 
   configuration = cfg;
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
-                                &shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
   GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect, NULL);
+                                   &handle_client_disconnect,
+                                   NULL);
   GNUNET_SERVER_add_handlers (server,
                               server_handlers);
-  cadet = GNUNET_CADET_connect (cfg, NULL,
-                                &channel_new_cb,
+  _GSS_statistics = GNUNET_STATISTICS_create ("set", cfg);
+  cadet = GNUNET_CADET_connect (cfg,
+                                NULL,
                                 &channel_end_cb,
-                                cadet_handlers,
-                                cadet_ports);
+                                cadet_handlers);
   if (NULL == cadet)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,