-bugfixes, code cleanup
[oweals/gnunet.git] / src / dht / dht_api.c
index 820bf35c87aed88d282aaab4f47d3619060112e3..0905486d12cb508ffc3b4e02743b06e1a5f3dfa3 100644 (file)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2009, 2010 Christian Grothoff (and other contributing authors)
+     (C) 2009, 2010, 2011, 2012 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
  */
 
 #include "platform.h"
-#include "gnunet_bandwidth_lib.h"
-#include "gnunet_client_lib.h"
+#include "gnunet_util_lib.h"
 #include "gnunet_constants.h"
-#include "gnunet_container_lib.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_protocols.h"
-#include "gnunet_server_lib.h"
-#include "gnunet_time_lib.h"
 #include "gnunet_dht_service.h"
 #include "dht.h"
 
-#define DEBUG_DHT_API GNUNET_NO
+#define LOG(kind,...) GNUNET_log_from (kind, "dht-api",__VA_ARGS__)
 
 /**
  * Entry in our list of messages to be (re-)transmitted.
@@ -60,12 +56,12 @@ struct PendingMessage
    * of this struct.
    */
   const struct GNUNET_MessageHeader *msg;
-  
+
   /**
    * Handle to the DHT API context.
    */
   struct GNUNET_DHT_Handle *handle;
-                       
+
   /**
    * Continuation to call when the request has been
    * transmitted (for the first time) to the service; can be NULL.
@@ -77,11 +73,6 @@ struct PendingMessage
    */
   void *cont_cls;
 
-  /**
-   * Timeout task for this message
-   */
-  GNUNET_SCHEDULER_TaskIdentifier timeout_task;
-
   /**
    * Unique ID for this request
    */
@@ -104,15 +95,65 @@ struct PendingMessage
 
 
 /**
- * Handle to a route request
+ * Handle to a PUT request.
  */
-struct GNUNET_DHT_RouteHandle
+struct GNUNET_DHT_PutHandle
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct GNUNET_DHT_PutHandle *next;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct GNUNET_DHT_PutHandle *prev;
+
+  /**
+   * Continuation to call when done.
+   */
+  GNUNET_DHT_PutContinuation cont;
+
+  /**
+   * Pending message associated with this PUT operation, 
+   * NULL after the message has been transmitted to the service.
+   */
+  struct PendingMessage *pending;
+
+  /**
+   * Main handle to this DHT api
+   */
+  struct GNUNET_DHT_Handle *dht_handle;
+
+  /**
+   * Closure for 'cont'.
+   */
+  void *cont_cls;
+
+  /**
+   * Timeout task for this operation.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier timeout_task;
+
+  /**
+   * Unique ID for the PUT operation.
+   */
+  uint64_t unique_id;
+
+};
+
+
+
+/**
+ * Handle to a GET request
+ */
+struct GNUNET_DHT_GetHandle
 {
 
   /**
    * Iterator to call on data receipt
    */
-  GNUNET_DHT_ReplyProcessor iter;
+  GNUNET_DHT_GetIterator iter;
 
   /**
    * Closure for the iterator callback
@@ -134,16 +175,69 @@ struct GNUNET_DHT_RouteHandle
   /**
    * Key that this get request is for
    */
-  GNUNET_HashCode key;
+  struct GNUNET_HashCode key;
 
   /**
-   * Unique identifier for this request (for key collisions). FIXME: redundant!?
+   * Unique identifier for this request (for key collisions).
    */
-  uint64_t uid;
+  uint64_t unique_id;
 
 };
 
 
+/**
+ * Handle to a monitoring request.
+ */
+struct GNUNET_DHT_MonitorHandle
+{
+  /**
+   * DLL.
+   */
+  struct GNUNET_DHT_MonitorHandle *next;
+
+  /**
+   * DLL.
+   */
+  struct GNUNET_DHT_MonitorHandle *prev;
+  
+  /**
+   * Main handle to this DHT api.
+   */
+  struct GNUNET_DHT_Handle *dht_handle;
+
+  /**
+   * Type of block looked for.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Key being looked for, NULL == all.
+   */
+  struct GNUNET_HashCode *key;
+
+  /**
+   * Callback for each received message of type get.
+   */
+  GNUNET_DHT_MonitorGetCB get_cb;
+
+  /**
+   * Callback for each received message of type get response.
+   */
+  GNUNET_DHT_MonitorGetRespCB get_resp_cb;
+
+  /**
+   * Callback for each received message of type put.
+   */
+  GNUNET_DHT_MonitorPutCB put_cb;
+
+  /**
+   * Closure for cb.
+   */
+  void *cb_cls;
+  
+};
+
+
 /**
  * Connection to the DHT service.
  */
@@ -176,8 +270,28 @@ struct GNUNET_DHT_Handle
   struct PendingMessage *pending_tail;
 
   /**
-   * Hash map containing the current outstanding unique requests
-   * (values are of type 'struct GNUNET_DHT_RouteHandle').
+   * Head of linked list of messages we would like to monitor. 
+   */
+  struct GNUNET_DHT_MonitorHandle *monitor_head;
+
+  /**
+   * Tail of linked list of messages we would like to monitor.
+   */
+  struct GNUNET_DHT_MonitorHandle *monitor_tail;
+
+  /**
+   * Head of active PUT requests.
+   */
+  struct GNUNET_DHT_PutHandle *put_head;
+
+  /**
+   * Tail of active PUT requests.
+   */
+  struct GNUNET_DHT_PutHandle *put_tail;
+
+  /**
+   * Hash map containing the current outstanding unique GET requests
+   * (values are of type 'struct GNUNET_DHT_GetHandle').
    */
   struct GNUNET_CONTAINER_MultiHashMap *active_requests;
 
@@ -197,53 +311,43 @@ struct GNUNET_DHT_Handle
    */
   uint64_t uid_gen;
 
+  /**
+   * Did we start our receive loop yet?
+   */
+  int in_receive;
 };
 
 
-/**
- * Transmit the next pending message, called by notify_transmit_ready
- */
-static size_t
-transmit_pending (void *cls,
-                 size_t size, 
-                 void *buf);
-
-
 /**
  * Handler for messages received from the DHT service
  * a demultiplexer which handles numerous message types
  *
+ * @param cls the 'struct GNUNET_DHT_Handle'
+ * @param msg the incoming message
  */
 static void
-service_message_handler (void *cls,
-                         const struct GNUNET_MessageHeader *msg);
-
-
+service_message_handler (void *cls, const struct GNUNET_MessageHeader *msg);
 
 
 /**
  * Try to (re)connect to the DHT service.
  *
+ * @param handle DHT handle to reconnect
  * @return GNUNET_YES on success, GNUNET_NO on failure.
  */
 static int
 try_connect (struct GNUNET_DHT_Handle *handle)
 {
-  if (handle->client != NULL)
+  if (NULL != handle->client)
     return GNUNET_OK;
+  handle->in_receive = GNUNET_NO;
   handle->client = GNUNET_CLIENT_connect ("dht", handle->cfg);
-  if (handle->client == NULL)
-    { 
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 _("Failed to connect to the DHT service!\n"));
-      return GNUNET_NO;
-    }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Starting to process replies from DHT\n");
-  GNUNET_CLIENT_receive (handle->client,
-                         &service_message_handler,
-                         handle, 
-                        GNUNET_TIME_UNIT_FOREVER_REL);
+  if (NULL == handle->client)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Failed to connect to the DHT service!\n"));
+    return GNUNET_NO;
+  }
   return GNUNET_YES;
 }
 
@@ -254,28 +358,37 @@ try_connect (struct GNUNET_DHT_Handle *handle)
  *
  * @param cls the 'struct GNUNET_DHT_Handle*'
  * @param key key for the request (not used)
- * @param value the 'struct GNUNET_DHT_RouteHandle*'
+ * @param value the 'struct GNUNET_DHT_GetHandle*'
  * @return GNUNET_YES (always)
  */
 static int
-add_request_to_pending (void *cls,
-                       const GNUNET_HashCode *key,
-                       void *value)
+add_request_to_pending (void *cls, const struct GNUNET_HashCode * key, void *value)
 {
   struct GNUNET_DHT_Handle *handle = cls;
-  struct GNUNET_DHT_RouteHandle *rh = value;
+  struct GNUNET_DHT_GetHandle *rh = value;
 
   if (GNUNET_NO == rh->message->in_pending_queue)
-    {
-      GNUNET_CONTAINER_DLL_insert (handle->pending_head,
-                                  handle->pending_tail,
-                                  rh->message);
-      rh->message->in_pending_queue = GNUNET_YES;
-    }
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Retransmitting request related to %s to DHT %p\n", GNUNET_h2s (key),
+         handle);
+    GNUNET_CONTAINER_DLL_insert (handle->pending_head, handle->pending_tail,
+                                 rh->message);
+    rh->message->in_pending_queue = GNUNET_YES;
+  }
   return GNUNET_YES;
 }
 
 
+/**
+ * Try to send messages from list of messages to send
+ *
+ * @param handle DHT_Handle
+ */
+static void
+process_pending_messages (struct GNUNET_DHT_Handle *handle);
+
+
 /**
  * Try reconnecting to the dht service.
  *
@@ -283,38 +396,21 @@ add_request_to_pending (void *cls,
  * @param tc scheduler context
  */
 static void
-try_reconnect (void *cls,
-               const struct GNUNET_SCHEDULER_TaskContext *tc)
+try_reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_DHT_Handle *handle = cls;
-  handle->reconnect_task = GNUNET_SCHEDULER_NO_TASK;
-  if (handle->retry_time.rel_value < GNUNET_CONSTANTS_SERVICE_RETRY.rel_value)
-    handle->retry_time = GNUNET_CONSTANTS_SERVICE_RETRY;
-  else
-    handle->retry_time = GNUNET_TIME_relative_multiply (handle->retry_time, 2);
-  if (handle->retry_time.rel_value > GNUNET_CONSTANTS_SERVICE_TIMEOUT.rel_value)
-    handle->retry_time = GNUNET_CONSTANTS_SERVICE_TIMEOUT;
-  handle->reconnect_task = GNUNET_SCHEDULER_NO_TASK;
-  handle->client = GNUNET_CLIENT_connect ("dht", handle->cfg);
-  if (handle->client == NULL)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "dht reconnect failed(!)\n");
-      return;
-    }
 
-  GNUNET_CONTAINER_multihashmap_iterate (handle->active_requests,
-                                         &add_request_to_pending,
-                                         handle);
-  if (handle->pending_head == NULL)
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Reconnecting with DHT %p\n", handle);
+  handle->retry_time = GNUNET_TIME_STD_BACKOFF (handle->retry_time);
+  handle->reconnect_task = GNUNET_SCHEDULER_NO_TASK;
+  if (GNUNET_YES != try_connect (handle))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "dht reconnect failed(!)\n");
     return;
-
-  GNUNET_CLIENT_notify_transmit_ready (handle->client,
-                                       ntohs(handle->pending_head->msg->size),
-                                       GNUNET_TIME_UNIT_FOREVER_REL,
-                                       GNUNET_NO,
-                                       &transmit_pending,
-                                       handle);
+  }
+  GNUNET_CONTAINER_multihashmap_iterate (handle->active_requests,
+                                         &add_request_to_pending, handle);
+  process_pending_messages (handle);
 }
 
 
@@ -326,100 +422,141 @@ try_reconnect (void *cls,
 static void
 do_disconnect (struct GNUNET_DHT_Handle *handle)
 {
-  if (handle->client == NULL)
+  struct GNUNET_DHT_PutHandle *ph;
+  struct GNUNET_DHT_PutHandle *next;
+
+  if (NULL == handle->client)
     return;
-  GNUNET_assert(handle->reconnect_task == GNUNET_SCHEDULER_NO_TASK);
-  GNUNET_CLIENT_disconnect (handle->client, GNUNET_NO);
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == handle->reconnect_task);
+  if (NULL != handle->th)
+    GNUNET_CLIENT_notify_transmit_ready_cancel (handle->th);
+  handle->th = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Disconnecting from DHT service, will try to reconnect in %s\n",
+              GNUNET_STRINGS_relative_time_to_string (handle->retry_time,
+                                                     GNUNET_YES));
+  GNUNET_CLIENT_disconnect (handle->client);
   handle->client = NULL;
-  handle->reconnect_task = GNUNET_SCHEDULER_add_delayed (handle->retry_time,
-                                                         &try_reconnect,
-                                                         handle);
+
+  /* signal disconnect to all PUT requests that were transmitted but waiting
+     for the put confirmation */
+  next = handle->put_head;
+  while (NULL != (ph = next))
+  {
+    next = ph->next;
+    if (NULL == ph->pending)
+    {
+      if (NULL != ph->cont)
+       ph->cont (ph->cont_cls, GNUNET_SYSERR);
+      GNUNET_DHT_put_cancel (ph);
+    }
+  }
+  handle->reconnect_task =
+      GNUNET_SCHEDULER_add_delayed (handle->retry_time, &try_reconnect, handle);
 }
 
 
+/**
+ * Transmit the next pending message, called by notify_transmit_ready
+ *
+ * @param cls the DHT handle
+ * @param size number of bytes available in 'buf' for transmission
+ * @param buf where to copy messages for the service
+ * @return number of bytes written to 'buf'
+ */
+static size_t
+transmit_pending (void *cls, size_t size, void *buf);
+
+
 /**
  * Try to send messages from list of messages to send
+ *
+ * @param handle handle to DHT
  */
 static void
 process_pending_messages (struct GNUNET_DHT_Handle *handle)
 {
   struct PendingMessage *head;
 
-  if (handle->client == NULL)
-    {
-      do_disconnect(handle);
-      return;
-    }
-  if (handle->th != NULL)
+  if (NULL == handle->client)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "process_pending_messages called, but client is NULL, reconnecting\n");
+    do_disconnect (handle);
+    return;
+  }
+  if (NULL != handle->th)
     return;
   if (NULL == (head = handle->pending_head))
     return;
-  handle->th = GNUNET_CLIENT_notify_transmit_ready (handle->client,
-                                                   ntohs (head->msg->size),
-                                                   GNUNET_TIME_UNIT_FOREVER_REL, 
-                                                   GNUNET_YES,
-                                                   &transmit_pending,
-                                                   handle);
-  if (NULL == handle->th)    
-    {
-      do_disconnect (handle);
-      return;
-    }
+  handle->th =
+      GNUNET_CLIENT_notify_transmit_ready (handle->client,
+                                           ntohs (head->msg->size),
+                                           GNUNET_TIME_UNIT_FOREVER_REL,
+                                           GNUNET_YES, &transmit_pending,
+                                           handle);
+  if (NULL != handle->th)
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "notify_transmit_ready returned NULL, reconnecting\n");
+  do_disconnect (handle);
 }
 
 
 /**
  * Transmit the next pending message, called by notify_transmit_ready
+ *
+ * @param cls the DHT handle
+ * @param size number of bytes available in 'buf' for transmission
+ * @param buf where to copy messages for the service
+ * @return number of bytes written to 'buf'
  */
 static size_t
-transmit_pending (void *cls,
-                 size_t size, 
-                 void *buf)
+transmit_pending (void *cls, size_t size, void *buf)
 {
   struct GNUNET_DHT_Handle *handle = cls;
   struct PendingMessage *head;
   size_t tsize;
 
   handle->th = NULL;
-  if (buf == NULL)
-    {
-      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Transmission to DHT service failed!  Reconnecting!\n");
-      do_disconnect (handle);
-      return 0;
-    }
+  if (NULL == buf)
+  {    
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Transmission to DHT service failed!  Reconnecting!\n");
+    do_disconnect (handle);
+    return 0;
+  }
   if (NULL == (head = handle->pending_head))
     return 0;
-  
+
   tsize = ntohs (head->msg->size);
   if (size < tsize)
-    {
-      process_pending_messages (handle);
-      return 0;
-    }
+  {
+    process_pending_messages (handle);
+    return 0;
+  }
   memcpy (buf, head->msg, tsize);
-  GNUNET_CONTAINER_DLL_remove (handle->pending_head,
-                              handle->pending_tail,
-                              head);
-  if (head->timeout_task != GNUNET_SCHEDULER_NO_TASK)
-    {
-      GNUNET_SCHEDULER_cancel (head->timeout_task);
-      head->timeout_task = GNUNET_SCHEDULER_NO_TASK;
-    }
-  if (NULL != head->cont)
-    {
-      GNUNET_SCHEDULER_add_continuation (head->cont,
-                                         head->cont_cls,
-                                         GNUNET_SCHEDULER_REASON_PREREQ_DONE);
-      head->cont = NULL;
-      head->cont_cls = NULL;
-    }
+  GNUNET_CONTAINER_DLL_remove (handle->pending_head, handle->pending_tail,
+                               head);
   head->in_pending_queue = GNUNET_NO;
+  if (NULL != head->cont)
+  {
+    head->cont (head->cont_cls, NULL);
+    head->cont = NULL;
+    head->cont_cls = NULL;
+  }
   if (GNUNET_YES == head->free_on_send)
     GNUNET_free (head);
   process_pending_messages (handle);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Forwarded request of %u bytes to DHT service\n",
-             (unsigned int) tsize);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Forwarded request of %u bytes to DHT service\n", (unsigned int) tsize);
+  if (GNUNET_NO == handle->in_receive)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "Starting to process replies from DHT\n");
+    handle->in_receive = GNUNET_YES;
+    GNUNET_CLIENT_receive (handle->client, &service_message_handler, handle,
+                           GNUNET_TIME_UNIT_FOREVER_REL);
+  }
   return tsize;
 }
 
@@ -427,68 +564,232 @@ transmit_pending (void *cls,
 /**
  * Process a given reply that might match the given
  * request.
+ *
+ * @param cls the 'struct GNUNET_DHT_ClientResultMessage'
+ * @param key query of the request
+ * @param value the 'struct GNUNET_DHT_RouteHandle' of a request matching the same key
+ * @return GNUNET_YES to continue to iterate over all results,
+ *         GNUNET_NO if the reply is malformed
  */
 static int
-process_reply (void *cls,
-              const GNUNET_HashCode *key,
-              void *value)
+process_reply (void *cls, const struct GNUNET_HashCode * key, void *value)
 {
-  const struct GNUNET_DHT_RouteResultMessage *dht_msg = cls;
-  struct GNUNET_DHT_RouteHandle *rh = value;
-  const struct GNUNET_MessageHeader *enc_msg;
-  size_t enc_size;
-  uint64_t uid;
-  const struct GNUNET_PeerIdentity **outgoing_path;
-  const struct GNUNET_PeerIdentity *pos;
-  uint32_t outgoing_path_length;
-  unsigned int i;
-  char *path_offset;
-
-  uid = GNUNET_ntohll (dht_msg->unique_id);
-  if (uid != rh->uid)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Reply UID did not match request UID\n");
-      return GNUNET_YES;
-    }
-  enc_msg = (const struct GNUNET_MessageHeader *)&dht_msg[1];
-  enc_size = ntohs (enc_msg->size);
-  if (enc_size < sizeof (struct GNUNET_MessageHeader))
-    {
-      GNUNET_break (0);
-      return GNUNET_NO;
-    }
-  path_offset = (char *)&dht_msg[1];
-  path_offset += enc_size;
-  pos = (const struct GNUNET_PeerIdentity *) path_offset;
-  outgoing_path_length = ntohl (dht_msg->outgoing_path_length);
-  if (outgoing_path_length * sizeof (struct GNUNET_PeerIdentity) > ntohs(dht_msg->header.size) - enc_size)
-    {
-      GNUNET_break (0);
-      return GNUNET_NO;
-    }
+  const struct GNUNET_DHT_ClientResultMessage *dht_msg = cls;
+  struct GNUNET_DHT_GetHandle *get_handle = value;
+  const struct GNUNET_PeerIdentity *put_path;
+  const struct GNUNET_PeerIdentity *get_path;
+  uint32_t put_path_length;
+  uint32_t get_path_length;
+  size_t data_length;
+  size_t msize;
+  size_t meta_length;
+  const void *data;
+
+  if (dht_msg->unique_id != get_handle->unique_id)
+  {
+    /* UID mismatch */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring reply for %s: UID mismatch: %llu/%llu\n", GNUNET_h2s (key),
+         dht_msg->unique_id, get_handle->unique_id);
+    return GNUNET_YES;
+  }
+  msize = ntohs (dht_msg->header.size);
+  put_path_length = ntohl (dht_msg->put_path_length);
+  get_path_length = ntohl (dht_msg->get_path_length);
+  meta_length =
+      sizeof (struct GNUNET_DHT_ClientResultMessage) +
+      sizeof (struct GNUNET_PeerIdentity) * (get_path_length + put_path_length);
+  if ((msize < meta_length) ||
+      (get_path_length >
+       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+      (put_path_length >
+       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
+  {
+    GNUNET_break (0);
+    return GNUNET_NO;
+  }
+  data_length = msize - meta_length;
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Giving %u byte reply for %s to application\n",
+       (unsigned int) data_length, GNUNET_h2s (key));
+  put_path = (const struct GNUNET_PeerIdentity *) &dht_msg[1];
+  get_path = &put_path[put_path_length];
+  data = &get_path[get_path_length];
+  get_handle->iter (get_handle->iter_cls,
+                    GNUNET_TIME_absolute_ntoh (dht_msg->expiration), key,
+                    get_path, get_path_length, put_path, put_path_length,
+                    ntohl (dht_msg->type), data_length, data);
+  return GNUNET_YES;
+}
 
-  if (outgoing_path_length > 0)
-    {
-      outgoing_path = GNUNET_malloc ((outgoing_path_length + 1) * sizeof (struct GNUNET_PeerIdentity*));
-      for (i = 0; i < outgoing_path_length; i++)
-       {
-         outgoing_path[i] = pos;
-         pos++;
-       }
-      outgoing_path[outgoing_path_length] = NULL;
-    }
-  else
-    outgoing_path = NULL;
+/**
+ * Process a get monitor message from the service.
+ *
+ * @param handle The DHT handle.
+ * @param msg Monitor get message from the service.
+ * 
+ * @return GNUNET_OK if everything went fine,
+ *         GNUNET_SYSERR if the message is malformed.
+ */
+static int
+process_monitor_get_message (struct GNUNET_DHT_Handle *handle,
+                             const struct GNUNET_DHT_MonitorGetMessage *msg)
+{
+  struct GNUNET_DHT_MonitorHandle *h;
+
+  for (h = handle->monitor_head; NULL != h; h = h->next)
+  {
+    int type_ok;
+    int key_ok;
+
+    type_ok = (GNUNET_BLOCK_TYPE_ANY == h->type) || (h->type == ntohl(msg->type));
+    key_ok = (NULL == h->key) || (0 == memcmp (h->key, &msg->key,
+                                              sizeof (struct GNUNET_HashCode)));
+    if (type_ok && key_ok && (NULL != h->get_cb))
+      h->get_cb (h->cb_cls,
+                ntohl (msg->options),
+                (enum GNUNET_BLOCK_Type) ntohl(msg->type),
+                ntohl (msg->hop_count),
+                ntohl (msg->desired_replication_level),
+                ntohl (msg->get_path_length),
+                (struct GNUNET_PeerIdentity *) &msg[1],
+                &msg->key);    
+  }
+  return GNUNET_OK;
+}
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Processing reply.\n");
-  rh->iter (rh->iter_cls, 
-           &rh->key,
-           outgoing_path,
-           enc_msg);
-  GNUNET_free_non_null (outgoing_path);
-  return GNUNET_YES;
+
+/**
+ * Process a get response monitor message from the service.
+ *
+ * @param handle The DHT handle.
+ * @param msg monitor get response message from the service
+ * @return GNUNET_OK if everything went fine,
+ *         GNUNET_SYSERR if the message is malformed.
+ */
+static int
+process_monitor_get_resp_message (struct GNUNET_DHT_Handle *handle,
+                                  const struct GNUNET_DHT_MonitorGetRespMessage
+                                  *msg)
+{
+  struct GNUNET_DHT_MonitorHandle *h;
+  struct GNUNET_PeerIdentity *path;
+  uint32_t getl;
+  uint32_t putl;
+  size_t msize;
+
+  msize = ntohs (msg->header.size);
+  path = (struct GNUNET_PeerIdentity *) &msg[1];
+  getl = ntohl (msg->get_path_length);
+  putl = ntohl (msg->put_path_length);
+  if ( (getl + putl < getl) ||
+       ( ((msize - sizeof (struct GNUNET_DHT_MonitorGetRespMessage)) / sizeof (struct GNUNET_PeerIdentity)) < getl + putl) )
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  for (h = handle->monitor_head; NULL != h; h = h->next)
+  {
+    int type_ok;
+    int key_ok;
+
+    type_ok = (GNUNET_BLOCK_TYPE_ANY == h->type) || (h->type == ntohl(msg->type));
+    key_ok = (NULL == h->key) || (0 == memcmp (h->key, &msg->key,
+                                              sizeof (struct GNUNET_HashCode)));
+    if (type_ok && key_ok && (NULL != h->get_resp_cb))
+      h->get_resp_cb (h->cb_cls,
+                      (enum GNUNET_BLOCK_Type) ntohl(msg->type),
+                      path, getl,
+                      &path[getl], putl,
+                      GNUNET_TIME_absolute_ntoh(msg->expiration_time),
+                      &msg->key,
+                      (void *) &path[getl + putl],
+                      msize -
+                      sizeof (struct GNUNET_DHT_MonitorGetRespMessage) -
+                      sizeof (struct GNUNET_PeerIdentity) * (putl + getl));
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a put monitor message from the service.
+ *
+ * @param handle The DHT handle.
+ * @param msg Monitor put message from the service.
+ * 
+ * @return GNUNET_OK if everything went fine,
+ *         GNUNET_SYSERR if the message is malformed.
+ */
+static int
+process_monitor_put_message (struct GNUNET_DHT_Handle *handle,
+                             const struct GNUNET_DHT_MonitorPutMessage *msg)
+{
+  struct GNUNET_DHT_MonitorHandle *h;
+  size_t msize;
+  struct GNUNET_PeerIdentity *path;
+  uint32_t putl;
+
+  msize = ntohs (msg->header.size);
+  path = (struct GNUNET_PeerIdentity *) &msg[1];
+  putl = ntohl (msg->put_path_length);
+  if (((msize - sizeof (struct GNUNET_DHT_MonitorGetRespMessage)) / sizeof (struct GNUNET_PeerIdentity)) < putl)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  for (h = handle->monitor_head; NULL != h; h = h->next)
+  {
+    int type_ok;
+    int key_ok;
+
+    type_ok = (GNUNET_BLOCK_TYPE_ANY == h->type) || (h->type == ntohl(msg->type));
+    key_ok = (NULL == h->key) || (0 == memcmp (h->key, &msg->key,
+                                              sizeof (struct GNUNET_HashCode)));
+    if (type_ok && key_ok && (NULL != h->put_cb))
+      h->put_cb (h->cb_cls,
+                 ntohl (msg->options),
+                 (enum GNUNET_BLOCK_Type) ntohl(msg->type),
+                 ntohl (msg->hop_count),
+                 ntohl (msg->desired_replication_level),
+                 putl, path,
+                 GNUNET_TIME_absolute_ntoh(msg->expiration_time),
+                 &msg->key,
+                 (void *) &path[putl],
+                 msize -
+                 sizeof (struct GNUNET_DHT_MonitorPutMessage) -
+                 sizeof (struct GNUNET_PeerIdentity) * putl);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a put confirmation message from the service.
+ *
+ * @param handle The DHT handle.
+ * @param msg confirmation message from the service.
+ * @return GNUNET_OK if everything went fine,
+ *         GNUNET_SYSERR if the message is malformed.
+ */
+static int
+process_put_confirmation_message (struct GNUNET_DHT_Handle *handle,
+                                 const struct GNUNET_DHT_ClientPutConfirmationMessage *msg)
+{
+  struct GNUNET_DHT_PutHandle *ph;
+  GNUNET_DHT_PutContinuation cont;
+  void *cont_cls;
+
+  for (ph = handle->put_head; NULL != ph; ph = ph->next)
+    if (ph->unique_id == msg->unique_id)
+      break;
+  if (NULL == ph)
+    return GNUNET_OK;
+  cont = ph->cont;
+  cont_cls = ph->cont_cls;
+  GNUNET_DHT_put_cancel (ph);
+  if (NULL != cont) 
+    cont (cont_cls, GNUNET_OK);
+  return GNUNET_OK;
 }
 
 
@@ -500,46 +801,93 @@ process_reply (void *cls,
  * @param msg the incoming message
  */
 static void
-service_message_handler (void *cls,
-                         const struct GNUNET_MessageHeader *msg)
+service_message_handler (void *cls, const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_DHT_Handle *handle = cls;
-  const struct GNUNET_DHT_RouteResultMessage *dht_msg;
+  const struct GNUNET_DHT_ClientResultMessage *dht_msg;
+  uint16_t msize;
+  int ret;
 
-  if (msg == NULL)
+  if (NULL == msg)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Error receiving data from DHT service, reconnecting\n");
+    do_disconnect (handle);
+    return;
+  }
+  ret = GNUNET_SYSERR;
+  msize = ntohs (msg->size);
+  switch (ntohs (msg->type))
+  {
+  case GNUNET_MESSAGE_TYPE_DHT_MONITOR_GET:
+    if (msize < sizeof (struct GNUNET_DHT_MonitorGetMessage))
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Error receiving data from DHT service, reconnecting\n");
-      do_disconnect (handle);
-      return;
+      GNUNET_break (0);
+      break;
     }
-  if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_DHT_LOCAL_ROUTE_RESULT)
+    ret = process_monitor_get_message(handle,
+                                     (const struct GNUNET_DHT_MonitorGetMessage *) msg);
+    break;
+  case GNUNET_MESSAGE_TYPE_DHT_MONITOR_GET_RESP:
+    if (msize < sizeof (struct GNUNET_DHT_MonitorGetRespMessage))
     {
       GNUNET_break (0);
-      do_disconnect (handle);
-      return;
+      break;
     }
-  if (ntohs (msg->size) < sizeof (struct GNUNET_DHT_RouteResultMessage))
+    ret = process_monitor_get_resp_message(handle,
+                                          (const struct GNUNET_DHT_MonitorGetRespMessage *) msg);
+    break;
+  case GNUNET_MESSAGE_TYPE_DHT_MONITOR_PUT:
+    if (msize < sizeof (struct GNUNET_DHT_MonitorPutMessage))
     {
       GNUNET_break (0);
-      do_disconnect (handle);
-      return;
+      break;
     }
-  dht_msg = (const struct GNUNET_DHT_RouteResultMessage *) msg;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Comparing reply `%s' against %u pending requests.\n",
-             GNUNET_h2s (&dht_msg->key),
-             GNUNET_CONTAINER_multihashmap_size (handle->active_requests));
-  GNUNET_CONTAINER_multihashmap_get_multiple (handle->active_requests,
-                                             &dht_msg->key,
-                                             &process_reply,
-                                             (void*) dht_msg);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Continuing to process replies from DHT\n");
-  GNUNET_CLIENT_receive (handle->client,
-                         &service_message_handler,
-                         handle, GNUNET_TIME_UNIT_FOREVER_REL);
-
+    ret = process_monitor_put_message(handle,
+                                     (const struct GNUNET_DHT_MonitorPutMessage *) msg);
+    break;
+  case GNUNET_MESSAGE_TYPE_DHT_MONITOR_PUT_RESP:
+    /* Not implemented yet */
+    GNUNET_break(0);
+    break;
+  case GNUNET_MESSAGE_TYPE_DHT_CLIENT_RESULT:
+    if (ntohs (msg->size) < sizeof (struct GNUNET_DHT_ClientResultMessage))
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ret = GNUNET_OK;
+    dht_msg = (const struct GNUNET_DHT_ClientResultMessage *) msg;
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "Received reply for `%s' from DHT service %p\n",
+        GNUNET_h2s (&dht_msg->key), handle);
+    GNUNET_CONTAINER_multihashmap_get_multiple (handle->active_requests,
+                                               &dht_msg->key, &process_reply,
+                                               (void *) dht_msg);
+    break;
+  case GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK:
+    if (ntohs (msg->size) != sizeof (struct GNUNET_DHT_ClientPutConfirmationMessage))
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ret = process_put_confirmation_message (handle,
+                                           (const struct GNUNET_DHT_ClientPutConfirmationMessage*) msg);
+    break;
+  default:
+    GNUNET_break(0);
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Unknown DHT message type: %hu (%hu) size: %hu\n",
+         ntohs (msg->type), msg->type, msize);
+    break;
+  }
+  if (GNUNET_OK != ret)
+  {
+    GNUNET_break (0);
+    do_disconnect (handle);
+    return;
+  }
+  GNUNET_CLIENT_receive (handle->client, &service_message_handler, handle,
+                         GNUNET_TIME_UNIT_FOREVER_REL);
 }
 
 
@@ -560,13 +908,14 @@ GNUNET_DHT_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   handle = GNUNET_malloc (sizeof (struct GNUNET_DHT_Handle));
   handle->cfg = cfg;
-  handle->uid_gen = GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
-  handle->active_requests = GNUNET_CONTAINER_multihashmap_create (ht_len);
+  handle->uid_gen =
+      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
+  handle->active_requests = GNUNET_CONTAINER_multihashmap_create (ht_len, GNUNET_NO);
   if (GNUNET_NO == try_connect (handle))
-    {
-      GNUNET_DHT_disconnect (handle);
-      return NULL;
-    }
+  {
+    GNUNET_DHT_disconnect (handle);
+    return NULL;
+  }
   return handle;
 }
 
@@ -580,45 +929,47 @@ void
 GNUNET_DHT_disconnect (struct GNUNET_DHT_Handle *handle)
 {
   struct PendingMessage *pm;
-  GNUNET_assert(handle != NULL);
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size(handle->active_requests));
-  if (handle->th != NULL)
-    {
-      GNUNET_CLIENT_notify_transmit_ready_cancel (handle->th);
-      handle->th = NULL;
-    }
+  struct GNUNET_DHT_PutHandle *ph;
+
+  GNUNET_assert (NULL != handle);
+  GNUNET_assert (0 ==
+                 GNUNET_CONTAINER_multihashmap_size (handle->active_requests));
+  if (NULL != handle->th)
+  {
+    GNUNET_CLIENT_notify_transmit_ready_cancel (handle->th);
+    handle->th = NULL;
+  }
   while (NULL != (pm = handle->pending_head))
-    {
-      GNUNET_CONTAINER_DLL_remove (handle->pending_head,
-                                  handle->pending_tail,
-                                  pm);
-      GNUNET_assert (GNUNET_YES == pm->free_on_send);
-      if (GNUNET_SCHEDULER_NO_TASK != pm->timeout_task)
-       GNUNET_SCHEDULER_cancel (pm->timeout_task);
-      if (NULL != pm->cont)
-       GNUNET_SCHEDULER_add_continuation (pm->cont,
-                                          pm->cont_cls,
-                                          GNUNET_SCHEDULER_REASON_TIMEOUT);
-      pm->in_pending_queue = GNUNET_NO;
-      GNUNET_free (pm);
-    }
-  if (handle->client != NULL)
-    {
-      GNUNET_CLIENT_disconnect (handle->client, GNUNET_YES);
-      handle->client = NULL;
-    }  
-  if (handle->reconnect_task != GNUNET_SCHEDULER_NO_TASK)
-    GNUNET_SCHEDULER_cancel(handle->reconnect_task);
-  GNUNET_CONTAINER_multihashmap_destroy(handle->active_requests);
+  {
+    GNUNET_assert (GNUNET_YES == pm->in_pending_queue);
+    GNUNET_CONTAINER_DLL_remove (handle->pending_head, handle->pending_tail,
+                                 pm);
+    pm->in_pending_queue = GNUNET_NO;
+    GNUNET_assert (GNUNET_YES == pm->free_on_send);
+    if (NULL != pm->cont)
+      pm->cont (pm->cont_cls, NULL);
+    GNUNET_free (pm);
+  }
+  while (NULL != (ph = handle->put_head))
+  {
+    GNUNET_break (NULL == ph->pending);
+    if (NULL != ph->cont)
+      ph->cont (ph->cont_cls, GNUNET_SYSERR);
+    GNUNET_DHT_put_cancel (ph);
+  }
+
+  if (NULL != handle->client)
+  {
+    GNUNET_CLIENT_disconnect (handle->client);
+    handle->client = NULL;
+  }
+  if (GNUNET_SCHEDULER_NO_TASK != handle->reconnect_task)
+    GNUNET_SCHEDULER_cancel (handle->reconnect_task);
+  GNUNET_CONTAINER_multihashmap_destroy (handle->active_requests);
   GNUNET_free (handle);
 }
 
 
-
-
-/* ***** Special low-level API providing generic routing abstraction ***** */
-
-
 /**
  * Timeout for the transmission of a fire&forget-request.  Clean it up.
  *
@@ -626,303 +977,386 @@ GNUNET_DHT_disconnect (struct GNUNET_DHT_Handle *handle)
  * @param tc scheduler context
  */
 static void
-timeout_route_request (void *cls,
+timeout_put_request (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_DHT_PutHandle *ph = cls;
+  struct GNUNET_DHT_Handle *handle = ph->dht_handle;
+
+  ph->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  if (NULL != ph->pending)
+  {
+    GNUNET_CONTAINER_DLL_remove (handle->pending_head, handle->pending_tail,
+                                ph->pending);
+    ph->pending->in_pending_queue = GNUNET_NO;
+    GNUNET_free (ph->pending);
+  }
+  if (NULL != ph->cont)
+    ph->cont (ph->cont_cls, GNUNET_NO);
+  GNUNET_CONTAINER_DLL_remove (handle->put_head,
+                              handle->put_tail,
+                              ph);
+  GNUNET_free (ph);
+}
+
+
+/**
+ * Function called whenever the PUT message leaves the queue.  Sets
+ * the message pointer in the put handle to NULL.
+ *
+ * @param cls the 'struct GNUNET_DHT_PutHandle'
+ * @param tc unused
+ */
+static void
+mark_put_message_gone (void *cls,
                       const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct PendingMessage *pending = cls;
-  struct GNUNET_DHT_Handle *handle;
+  struct GNUNET_DHT_PutHandle *ph = cls;
 
-  if (pending->free_on_send != GNUNET_YES)
-    {
-      /* timeouts should only apply to fire & forget requests! */
-      GNUNET_break (0);
-      return;
-    }
-  handle = pending->handle;
-  GNUNET_CONTAINER_DLL_remove (handle->pending_head,
-                              handle->pending_tail,
-                              pending);
-  if (pending->cont != NULL)
-    pending->cont (pending->cont_cls,
-                  tc);
-  GNUNET_free (pending);
+  ph->pending = NULL;
 }
 
 
 /**
- * Initiate a generic DHT route operation.
+ * Perform a PUT operation storing data in the DHT.  FIXME: we should
+ * change the protocol to get a confirmation for the PUT from the DHT
+ * and call 'cont' only after getting the confirmation; otherwise, the
+ * client has no good way of telling if the 'PUT' message actually got
+ * to the DHT service!
  *
- * @param handle handle to the DHT service
- * @param key the key to look up
- * @param desired_replication_level how many peers should ultimately receive
- *                this message (advisory only, target may be too high for the
- *                given DHT or not hit exactly).
- * @param options options for routing
- * @param enc send the encapsulated message to a peer close to the key
- * @param iter function to call on each result, NULL if no replies are expected
- * @param iter_cls closure for iter
- * @param timeout when to abort with an error if we fail to get
- *                a confirmation for the request (when necessary) or how long
- *                to wait for tramission to the service; only applies
- *                if 'iter' is NULL
- * @param cont continuation to call when the request has been transmitted
- *             the first time to the service
+ * @param handle handle to DHT service
+ * @param key the key to store under
+ * @param desired_replication_level estimate of how many
+ *                nearest peers this request should reach
+ * @param options routing options for this message
+ * @param type type of the value
+ * @param size number of bytes in data; must be less than 64k
+ * @param data the data to store
+ * @param exp desired expiration time for the value
+ * @param timeout how long to wait for transmission of this request
+ * @param cont continuation to call when done (transmitting request to service)
+ *        You must not call GNUNET_DHT_DISCONNECT in this continuation
  * @param cont_cls closure for cont
- * @return handle to stop the request, NULL if the request is "fire and forget"
  */
-struct GNUNET_DHT_RouteHandle *
-GNUNET_DHT_route_start (struct GNUNET_DHT_Handle *handle,
-                        const GNUNET_HashCode *key,
-                        uint32_t desired_replication_level,
-                        enum GNUNET_DHT_RouteOption options,
-                        const struct GNUNET_MessageHeader *enc,
-                        struct GNUNET_TIME_Relative timeout,
-                        GNUNET_DHT_ReplyProcessor iter,
-                        void *iter_cls,
-                       GNUNET_SCHEDULER_Task cont,
-                       void *cont_cls)
+struct GNUNET_DHT_PutHandle *
+GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle, const struct GNUNET_HashCode * key,
+                uint32_t desired_replication_level,
+                enum GNUNET_DHT_RouteOption options,
+                enum GNUNET_BLOCK_Type type, size_t size, const char *data,
+                struct GNUNET_TIME_Absolute exp,
+                struct GNUNET_TIME_Relative timeout, GNUNET_DHT_PutContinuation cont,
+                void *cont_cls)
 {
+  struct GNUNET_DHT_ClientPutMessage *put_msg;
+  size_t msize;
   struct PendingMessage *pending;
-  struct GNUNET_DHT_RouteMessage *message;
-  struct GNUNET_DHT_RouteHandle *route_handle;
-  uint16_t msize;
-  uint16_t esize;
-
-  esize = ntohs (enc->size);
-  if (sizeof (struct GNUNET_DHT_RouteMessage) + esize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-    {
-      GNUNET_break (0);
-      return NULL;
-    }
-  msize = sizeof (struct GNUNET_DHT_RouteMessage) + esize;
+  struct GNUNET_DHT_PutHandle *ph;
+
+  msize = sizeof (struct GNUNET_DHT_ClientPutMessage) + size;
+  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
+      (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  ph = GNUNET_malloc (sizeof (struct GNUNET_DHT_PutHandle));
+  ph->dht_handle = handle;
+  ph->timeout_task = GNUNET_SCHEDULER_add_delayed (timeout, &timeout_put_request, ph);
+  ph->cont = cont;
+  ph->cont_cls = cont_cls;
+  ph->unique_id = ++handle->uid_gen;
   pending = GNUNET_malloc (sizeof (struct PendingMessage) + msize);
-  message = (struct GNUNET_DHT_RouteMessage*) &pending[1];
-  pending->msg = &message->header;
+  ph->pending = pending;
+  put_msg = (struct GNUNET_DHT_ClientPutMessage *) &pending[1];
+  pending->msg = &put_msg->header;
   pending->handle = handle;
-  pending->cont = cont;
-  pending->cont_cls = cont_cls;
-  
-  message->header.size = htons (msize);
-  message->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_LOCAL_ROUTE);
-  message->key = *key;
-  message->options = htonl ((uint32_t) options);
-  message->desired_replication_level = htonl (desired_replication_level);
-  handle->uid_gen++;
-  message->unique_id = GNUNET_htonll (handle->uid_gen);
-  memcpy (&message[1], enc, esize);
-
-  if (iter != NULL)
-    {
-      route_handle = GNUNET_malloc (sizeof (struct GNUNET_DHT_RouteHandle));
-      route_handle->key = *key;
-      route_handle->iter = iter;
-      route_handle->iter_cls = iter_cls;
-      route_handle->dht_handle = handle;
-      route_handle->uid = handle->uid_gen;
-      route_handle->message = pending;
-      GNUNET_CONTAINER_multihashmap_put (handle->active_requests,
-                                         key,
-                                        route_handle,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-    }
-  else
-    {
-      route_handle = NULL;
-      pending->free_on_send = GNUNET_YES;
-      pending->timeout_task = GNUNET_SCHEDULER_add_delayed (timeout,
-                                                           &timeout_route_request,
-                                                           pending);
-    }
-  GNUNET_CONTAINER_DLL_insert (handle->pending_head,
-                              handle->pending_tail,
-                              pending);
+  pending->cont = &mark_put_message_gone;
+  pending->cont_cls = ph;
+  pending->free_on_send = GNUNET_YES;
+  put_msg->header.size = htons (msize);
+  put_msg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT);
+  put_msg->type = htonl (type);
+  put_msg->options = htonl ((uint32_t) options);
+  put_msg->desired_replication_level = htonl (desired_replication_level);
+  put_msg->unique_id = ph->unique_id;
+  put_msg->expiration = GNUNET_TIME_absolute_hton (exp);
+  put_msg->key = *key;
+  memcpy (&put_msg[1], data, size);
+  GNUNET_CONTAINER_DLL_insert (handle->pending_head, handle->pending_tail,
+                               pending);
   pending->in_pending_queue = GNUNET_YES;
+  GNUNET_CONTAINER_DLL_insert_tail (handle->put_head,
+                                   handle->put_tail,
+                                   ph);
   process_pending_messages (handle);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "DHT route start request processed, returning %p\n",
-             route_handle);
-  return route_handle;
+  return ph;
 }
 
 
 /**
- * Stop a previously issued routing request
+ * Cancels a DHT PUT operation.  Note that the PUT request may still
+ * go out over the network (we can't stop that); However, if the PUT
+ * has not yet been sent to the service, cancelling the PUT will stop
+ * this from happening (but there is no way for the user of this API
+ * to tell if that is the case).  The only use for this API is to 
+ * prevent a later call to 'cont' from "GNUNET_DHT_put" (i.e. because
+ * the system is shutting down).
  *
- * @param route_handle handle to the request to stop
+ * @param ph put operation to cancel ('cont' will no longer be called)
  */
 void
-GNUNET_DHT_route_stop (struct GNUNET_DHT_RouteHandle *route_handle)
+GNUNET_DHT_put_cancel (struct GNUNET_DHT_PutHandle *ph)
 {
-  struct GNUNET_DHT_Handle *handle;
-  struct PendingMessage *pending;
-  struct GNUNET_DHT_StopMessage *message;
-  size_t msize;
-
-  handle = route_handle->dht_handle;
-  if (GNUNET_NO == route_handle->message->in_pending_queue)
-    {
-      /* need to send stop message */
-      msize = sizeof (struct GNUNET_DHT_StopMessage);
-      pending = GNUNET_malloc (sizeof (struct PendingMessage) + 
-                              msize);
-      message = (struct GNUNET_DHT_StopMessage*) &pending[1];
-      pending->msg = &message->header;
-      message->header.size = htons (msize);
-      message->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_LOCAL_ROUTE_STOP);
-      message->unique_id = GNUNET_htonll (route_handle->uid);
-      message->key = route_handle->key;
-      pending->handle = handle;
-      pending->free_on_send = GNUNET_YES;
-      pending->in_pending_queue = GNUNET_YES;      
-      GNUNET_CONTAINER_DLL_insert (handle->pending_head,
-                                  handle->pending_tail,
-                                  pending);
-      process_pending_messages (handle);
-    }
-  else
-    {
-      /* simply remove pending request from message queue before
-        transmission, no need to transmit STOP request! */
-      GNUNET_CONTAINER_DLL_remove (handle->pending_head,
-                                  handle->pending_tail,
-                                  route_handle->message);
-    }
-  GNUNET_assert (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_remove (route_handle->dht_handle->active_requests,
-                                                      &route_handle->key,
-                                                      route_handle));
-  GNUNET_free(route_handle->message);
-  GNUNET_free(route_handle);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "DHT route stop request processed\n");
+  struct GNUNET_DHT_Handle *handle = ph->dht_handle;
+
+  if (NULL != ph->pending)
+  {
+    GNUNET_CONTAINER_DLL_remove (handle->pending_head,
+                                handle->pending_tail,
+                                ph->pending);
+    GNUNET_free (ph->pending);
+    ph->pending = NULL;
+  }
+  if (ph->timeout_task != GNUNET_SCHEDULER_NO_TASK)
+  {
+    GNUNET_SCHEDULER_cancel (ph->timeout_task);
+    ph->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  GNUNET_CONTAINER_DLL_remove (handle->put_head,
+                              handle->put_tail,
+                              ph);
+  GNUNET_free (ph);
 }
 
 
-
-/* ***** Special API for controlling DHT routing maintenance ******* */
-
-
 /**
- * Send a control message to the DHT.
+ * Perform an asynchronous GET operation on the DHT identified. See
+ * also "GNUNET_BLOCK_evaluate".
  *
  * @param handle handle to the DHT service
- * @param command command
- * @param variable variable to the command
- * @param cont continuation to call when done (transmitting request to service)
- * @param cont_cls closure for cont
+ * @param type expected type of the response object
+ * @param key the key to look up
+ * @param desired_replication_level estimate of how many
+                  nearest peers this request should reach
+ * @param options routing options for this message
+ * @param xquery extended query data (can be NULL, depending on type)
+ * @param xquery_size number of bytes in xquery
+ * @param iter function to call on each result
+ * @param iter_cls closure for iter
+ * @return handle to stop the async get
  */
-static void
-send_control_message (struct GNUNET_DHT_Handle *handle,
-                     uint16_t command,
-                     uint16_t variable,
-                     GNUNET_SCHEDULER_Task cont,
-                     void *cont_cls)
+struct GNUNET_DHT_GetHandle *
+GNUNET_DHT_get_start (struct GNUNET_DHT_Handle *handle,
+                      enum GNUNET_BLOCK_Type type, const struct GNUNET_HashCode * key,
+                      uint32_t desired_replication_level,
+                      enum GNUNET_DHT_RouteOption options, const void *xquery,
+                      size_t xquery_size, GNUNET_DHT_GetIterator iter,
+                      void *iter_cls)
 {
-  struct GNUNET_DHT_ControlMessage *msg;
+  struct GNUNET_DHT_ClientGetMessage *get_msg;
+  struct GNUNET_DHT_GetHandle *get_handle;
+  size_t msize;
   struct PendingMessage *pending;
 
-  pending = GNUNET_malloc (sizeof (struct PendingMessage) + 
-                          sizeof(struct GNUNET_DHT_ControlMessage)); 
-  msg = (struct GNUNET_DHT_ControlMessage*) &pending[1];
-  pending->msg = &msg->header;
-  msg->header.size = htons (sizeof(struct GNUNET_DHT_ControlMessage));
-  msg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CONTROL);
-  msg->command = htons (command);
-  msg->variable = htons (variable);
-  pending->free_on_send = GNUNET_YES;
-  pending->cont = cont;
-  pending->cont_cls = cont_cls;
-  pending->in_pending_queue = GNUNET_YES;      
-  GNUNET_CONTAINER_DLL_insert (handle->pending_head,
-                              handle->pending_tail,
-                              pending);
+  msize = sizeof (struct GNUNET_DHT_ClientGetMessage) + xquery_size;
+  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
+      (xquery_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sending query for %s to DHT %p\n",
+       GNUNET_h2s (key), handle);
+  pending = GNUNET_malloc (sizeof (struct PendingMessage) + msize);
+  get_msg = (struct GNUNET_DHT_ClientGetMessage *) &pending[1];
+  pending->msg = &get_msg->header;
+  pending->handle = handle;
+  pending->free_on_send = GNUNET_NO;
+  get_msg->header.size = htons (msize);
+  get_msg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET);
+  get_msg->options = htonl ((uint32_t) options);
+  get_msg->desired_replication_level = htonl (desired_replication_level);
+  get_msg->type = htonl (type);
+  get_msg->key = *key;
+  get_msg->unique_id = ++handle->uid_gen;
+  memcpy (&get_msg[1], xquery, xquery_size);
+  GNUNET_CONTAINER_DLL_insert (handle->pending_head, handle->pending_tail,
+                               pending);
+  pending->in_pending_queue = GNUNET_YES;
+  get_handle = GNUNET_malloc (sizeof (struct GNUNET_DHT_GetHandle));
+  get_handle->iter = iter;
+  get_handle->iter_cls = iter_cls;
+  get_handle->message = pending;
+  get_handle->unique_id = get_msg->unique_id;
+  GNUNET_CONTAINER_multihashmap_put (handle->active_requests, key, get_handle,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
   process_pending_messages (handle);
+  return get_handle;
 }
 
 
 /**
- * Send a message to the DHT telling it to issue a single find
- * peer request using the peers unique identifier as key.  This
- * is used to fill the routing table, and is normally controlled
- * by the DHT itself.  However, for testing and perhaps more
- * close control over the DHT, this can be explicitly managed.
+ * Stop async DHT-get.
  *
- * @param handle handle to the DHT service
- * @param cont continuation to call when done (transmitting request to service)
- * @param cont_cls closure for cont
+ * @param get_handle handle to the GET operation to stop
  */
 void
-GNUNET_DHT_find_peers (struct GNUNET_DHT_Handle *handle,
-                      GNUNET_SCHEDULER_Task cont,
-                      void *cont_cls)
+GNUNET_DHT_get_stop (struct GNUNET_DHT_GetHandle *get_handle)
 {
-  send_control_message (handle,
-                       GNUNET_MESSAGE_TYPE_DHT_FIND_PEER, 0,
-                       cont, cont_cls);
-}
-
+  struct GNUNET_DHT_Handle *handle;
+  const struct GNUNET_DHT_ClientGetMessage *get_msg;
+  struct GNUNET_DHT_ClientGetStopMessage *stop_msg;
+  struct PendingMessage *pending;
 
+  handle = get_handle->message->handle;
+  get_msg =
+      (const struct GNUNET_DHT_ClientGetMessage *) get_handle->message->msg;
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sending STOP for %s to DHT via %p\n",
+       GNUNET_h2s (&get_msg->key), handle);
+  /* generate STOP */
+  pending =
+      GNUNET_malloc (sizeof (struct PendingMessage) +
+                     sizeof (struct GNUNET_DHT_ClientGetStopMessage));
+  stop_msg = (struct GNUNET_DHT_ClientGetStopMessage *) &pending[1];
+  pending->msg = &stop_msg->header;
+  pending->handle = handle;
+  pending->free_on_send = GNUNET_YES;
+  stop_msg->header.size =
+      htons (sizeof (struct GNUNET_DHT_ClientGetStopMessage));
+  stop_msg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET_STOP);
+  stop_msg->reserved = htonl (0);
+  stop_msg->unique_id = get_msg->unique_id;
+  stop_msg->key = get_msg->key;
+  GNUNET_CONTAINER_DLL_insert (handle->pending_head, handle->pending_tail,
+                               pending);
+  pending->in_pending_queue = GNUNET_YES;
 
-#if HAVE_MALICIOUS
+  /* remove 'GET' from active status */
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (handle->active_requests,
+                                                       &get_msg->key,
+                                                       get_handle));
+  if (GNUNET_YES == get_handle->message->in_pending_queue)
+  {
+    GNUNET_CONTAINER_DLL_remove (handle->pending_head, handle->pending_tail,
+                                 get_handle->message);
+    get_handle->message->in_pending_queue = GNUNET_NO;
+  }
+  GNUNET_free (get_handle->message);
+  GNUNET_free (get_handle);
 
-/**
- * Send a message to the DHT telling it to start issuing random GET
- * requests every 'frequency' milliseconds.
- *
- * @param handle handle to the DHT service
- * @param frequency delay between sending malicious messages
- */
-void
-GNUNET_DHT_set_malicious_getter (struct GNUNET_DHT_Handle *handle,
-                                struct GNUNET_TIME_Relative frequency)
-{
-  if (frequency.rel_value > UINT16_MAX)
-    {
-      GNUNET_break (0);
-      return;
-    }
-  send_control_message (handle,
-                       GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET, frequency.rel_value,
-                       NULL, NULL);
+  process_pending_messages (handle);
 }
 
+
 /**
- * Send a message to the DHT telling it to start issuing random PUT
- * requests every 'frequency' milliseconds.
+ * Start monitoring the local DHT service.
  *
- * @param handle handle to the DHT service
- * @param frequency delay between sending malicious messages
+ * @param handle Handle to the DHT service.
+ * @param type Type of blocks that are of interest.
+ * @param key Key of data of interest, NULL for all.
+ * @param get_cb Callback to process monitored get messages.
+ * @param get_resp_cb Callback to process monitored get response messages.
+ * @param put_cb Callback to process monitored put messages.
+ * @param cb_cls Closure for cb.
+ *
+ * @return Handle to stop monitoring.
  */
-void 
-GNUNET_DHT_set_malicious_putter (struct GNUNET_DHT_Handle *handle, 
-                                struct GNUNET_TIME_Relative frequency)
+struct GNUNET_DHT_MonitorHandle *
+GNUNET_DHT_monitor_start (struct GNUNET_DHT_Handle *handle,
+                          enum GNUNET_BLOCK_Type type,
+                          const struct GNUNET_HashCode *key,
+                          GNUNET_DHT_MonitorGetCB get_cb,
+                          GNUNET_DHT_MonitorGetRespCB get_resp_cb,
+                          GNUNET_DHT_MonitorPutCB put_cb,
+                          void *cb_cls)
 {
-  if (frequency.rel_value > UINT16_MAX)
-    {
-      GNUNET_break (0);
-      return;
-    }
+  struct GNUNET_DHT_MonitorHandle *h;
+  struct GNUNET_DHT_MonitorStartStopMessage *m;
+  struct PendingMessage *pending;
+
+  h = GNUNET_malloc (sizeof (struct GNUNET_DHT_MonitorHandle));
+  GNUNET_CONTAINER_DLL_insert(handle->monitor_head, handle->monitor_tail, h);
+
+  h->get_cb = get_cb;
+  h->get_resp_cb = get_resp_cb;
+  h->put_cb = put_cb;
+  h->cb_cls = cb_cls;
+  h->type = type;
+  h->dht_handle = handle;
+  if (NULL != key)
+  {
+    h->key = GNUNET_malloc (sizeof(struct GNUNET_HashCode));
+    memcpy (h->key, key, sizeof(struct GNUNET_HashCode));
+  }
+
+  pending = GNUNET_malloc (sizeof (struct GNUNET_DHT_MonitorStartStopMessage) +
+                           sizeof (struct PendingMessage));
+  m = (struct GNUNET_DHT_MonitorStartStopMessage *) &pending[1];
+  pending->msg = &m->header;
+  pending->handle = handle;
+  pending->free_on_send = GNUNET_YES;
+  m->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_MONITOR_START);
+  m->header.size = htons (sizeof (struct GNUNET_DHT_MonitorStartStopMessage));
+  m->type = htonl(type);
+  m->get = htons(NULL != get_cb);
+  m->get_resp = htons(NULL != get_resp_cb);
+  m->put = htons(NULL != put_cb);
+  if (NULL != key) {
+    m->filter_key = htons(1);
+    memcpy (&m->key, key, sizeof(struct GNUNET_HashCode));
+  }
+  GNUNET_CONTAINER_DLL_insert (handle->pending_head, handle->pending_tail,
+                               pending);
+  pending->in_pending_queue = GNUNET_YES;
+  process_pending_messages (handle);
 
-  send_control_message (handle,
-                       GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT, frequency.rel_value,
-                       NULL, NULL);
+  return h;
 }
 
 
 /**
- * Send a message to the DHT telling it to start dropping
- * all requests received.
+ * Stop monitoring.
  *
- * @param handle handle to the DHT service
+ * @param handle The handle to the monitor request returned by monitor_start.
+ *
+ * On return get_handle will no longer be valid, caller must not use again!!!
  */
-void 
-GNUNET_DHT_set_malicious_dropper (struct GNUNET_DHT_Handle *handle)
+void
+GNUNET_DHT_monitor_stop (struct GNUNET_DHT_MonitorHandle *handle)
 {
-  send_control_message (handle,
-                       GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP, 0,
-                       NULL, NULL);
+  struct GNUNET_DHT_MonitorStartStopMessage *m;
+  struct PendingMessage *pending;
+
+  GNUNET_CONTAINER_DLL_remove (handle->dht_handle->monitor_head,
+                               handle->dht_handle->monitor_tail,
+                               handle);
+
+  pending = GNUNET_malloc (sizeof (struct GNUNET_DHT_MonitorStartStopMessage) +
+                           sizeof (struct PendingMessage));
+  m = (struct GNUNET_DHT_MonitorStartStopMessage *) &pending[1];
+  pending->msg = &m->header;
+  pending->handle = handle->dht_handle;
+  pending->free_on_send = GNUNET_YES;
+  m->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_MONITOR_STOP);
+  m->header.size = htons (sizeof (struct GNUNET_DHT_MonitorStartStopMessage));
+  m->type = htonl(handle->type);
+  m->get = htons(NULL != handle->get_cb);
+  m->get_resp = htons(NULL != handle->get_resp_cb);
+  m->put = htons(NULL != handle->put_cb);
+  if (NULL != handle->key) {
+    m->filter_key = htons(1);
+    memcpy (&m->key, handle->key, sizeof(struct GNUNET_HashCode));
+  }
+  GNUNET_CONTAINER_DLL_insert (handle->dht_handle->pending_head,
+                               handle->dht_handle->pending_tail,
+                               pending);
+  pending->in_pending_queue = GNUNET_YES;
+  process_pending_messages (handle->dht_handle);
+  
+  GNUNET_free_non_null (handle->key);
+  GNUNET_free (handle);
 }
 
-#endif
+
 
 /* end of dht_api.c */