Fix DTLS buffered message DoS attack

DTLS can handle out of order record delivery. Additionally since
handshake messages can be bigger than will fit into a single packet, the
messages can be fragmented across multiple records (as with normal TLS).
That means that the messages can arrive mixed up, and we have to
reassemble them. We keep a queue of buffered messages that are "from the
future", i.e. messages we're not ready to deal with yet but have arrived
early. The messages held there may not be full yet - they could be one
or more fragments that are still in the process of being reassembled.

The code assumes that we will eventually complete the reassembly and
when that occurs the complete message is removed from the queue at the
point that we need to use it.

However, DTLS is also tolerant of packet loss. To get around that DTLS
messages can be retransmitted. If we receive a full (non-fragmented)
message from the peer after previously having received a fragment of
that message, then we ignore the message in the queue and just use the
non-fragmented version. At that point the queued message will never get
removed.

Additionally the peer could send "future" messages that we never get to
in order to complete the handshake. Each message has a sequence number
(starting from 0). We will accept a message fragment for the current
message sequence number, or for any sequence up to 10 into the future.
However if the Finished message has a sequence number of 2, anything
greater than that in the queue is just left there.

So, in those two ways we can end up with "orphaned" data in the queue
that will never get removed - except when the connection is closed. At
that point all the queues are flushed.

An attacker could seek to exploit this by filling up the queues with
lots of large messages that are never going to be used in order to
attempt a DoS by memory exhaustion.

I will assume that we are only concerned with servers here. It does not
seem reasonable to be concerned about a memory exhaustion attack on a
client. They are unlikely to process enough connections for this to be
an issue.

A "long" handshake with many messages might be 5 messages long (in the
incoming direction), e.g. ClientHello, Certificate, ClientKeyExchange,
CertificateVerify, Finished. So this would be message sequence numbers 0
to 4. Additionally we can buffer up to 10 messages in the future.
Therefore the maximum number of messages that an attacker could send
that could get orphaned would typically be 15.

The maximum size that a DTLS message is allowed to be is defined by
max_cert_list, which by default is 100k. Therefore the maximum amount of
"orphaned" memory per connection is 1500k.

Message sequence numbers get reset after the Finished message, so
renegotiation will not extend the maximum number of messages that can be
orphaned per connection.

As noted above, the queues do get cleared when the connection is closed.
Therefore in order to mount an effective attack, an attacker would have
to open many simultaneous connections.

Issue reported by Quan Luo.

CVE-2016-2179

Reviewed-by: Richard Levitte <levitte@openssl.org>

diff --git a/ssl/d1_both.c b/ssl/d1_both.c
index 5d2c2093b2a9fee8c8456de9ce234a8f8f6f7547..46c70d8ad54c931827021c3560031a7aefa19c16 100644 (file)
--- a/ssl/d1_both.c
+++ b/ssl/d1_both.c
@@ -618,11 +618,23 @@ static int dtls1_retrieve_buffered_fragment(SSL *s, long max, int *ok)
     int al;
 
     *ok = 0;
-    item = pqueue_peek(s->d1->buffered_messages);
-    if (item == NULL)
-        return 0;
+    do {
+        item = pqueue_peek(s->d1->buffered_messages);
+        if (item == NULL)
+            return 0;
+
+        frag = (hm_fragment *)item->data;
+
+        if (frag->msg_header.seq < s->d1->handshake_read_seq) {
+            /* This is a stale message that has been buffered so clear it */
+            pqueue_pop(s->d1->buffered_messages);
+            dtls1_hm_fragment_free(frag);
+            pitem_free(item);
+            item = NULL;
+            frag = NULL;
+        }
+    } while (item == NULL);
 
-    frag = (hm_fragment *)item->data;
 
     /* Don't return if reassembly still in progress */
     if (frag->reassembly != NULL)
@@ -1296,18 +1308,6 @@ dtls1_retransmit_message(SSL *s, unsigned short seq, unsigned long frag_off,
     return ret;
 }
 
-/* call this function when the buffered messages are no longer needed */
-void dtls1_clear_record_buffer(SSL *s)
-{
-    pitem *item;
-
-    for (item = pqueue_pop(s->d1->sent_messages);
-         item != NULL; item = pqueue_pop(s->d1->sent_messages)) {
-        dtls1_hm_fragment_free((hm_fragment *)item->data);
-        pitem_free(item);
-    }
-}
-
 unsigned char *dtls1_set_message_header(SSL *s, unsigned char *p,
                                         unsigned char mt, unsigned long len,
                                         unsigned long frag_off,

diff --git a/ssl/d1_clnt.c b/ssl/d1_clnt.c
index 3ddfa7bca4b79e2b9d7c81e84fcf8738a92807c5..7e2f5c2830b5120e9f2bf6979aab1d4d8583b1ea 100644 (file)
--- a/ssl/d1_clnt.c
+++ b/ssl/d1_clnt.c
@@ -769,6 +769,7 @@ int dtls1_connect(SSL *s)
             /* done with handshaking */
             s->d1->handshake_read_seq = 0;
             s->d1->next_handshake_write_seq = 0;
+            dtls1_clear_received_buffer(s);
             goto end;
             /* break; */
 

diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index ee78921ba8b0d3d5ac6a75aa46601b79f37137f9..debd4fd5dcca40a2ecf9615845821a6f949c1c4d 100644 (file)
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -170,7 +170,6 @@ int dtls1_new(SSL *s)
 static void dtls1_clear_queues(SSL *s)
 {
     pitem *item = NULL;
-    hm_fragment *frag = NULL;
     DTLS1_RECORD_DATA *rdata;
 
     while ((item = pqueue_pop(s->d1->unprocessed_rcds.q)) != NULL) {
@@ -191,28 +190,44 @@ static void dtls1_clear_queues(SSL *s)
         pitem_free(item);
     }
 
+    while ((item = pqueue_pop(s->d1->buffered_app_data.q)) != NULL) {
+        rdata = (DTLS1_RECORD_DATA *)item->data;
+        if (rdata->rbuf.buf) {
+            OPENSSL_free(rdata->rbuf.buf);
+        }
+        OPENSSL_free(item->data);
+        pitem_free(item);
+    }
+
+    dtls1_clear_received_buffer(s);
+    dtls1_clear_sent_buffer(s);
+}
+
+void dtls1_clear_received_buffer(SSL *s)
+{
+    pitem *item = NULL;
+    hm_fragment *frag = NULL;
+
     while ((item = pqueue_pop(s->d1->buffered_messages)) != NULL) {
         frag = (hm_fragment *)item->data;
         dtls1_hm_fragment_free(frag);
         pitem_free(item);
     }
+}
+
+void dtls1_clear_sent_buffer(SSL *s)
+{
+    pitem *item = NULL;
+    hm_fragment *frag = NULL;
 
     while ((item = pqueue_pop(s->d1->sent_messages)) != NULL) {
         frag = (hm_fragment *)item->data;
         dtls1_hm_fragment_free(frag);
         pitem_free(item);
     }
-
-    while ((item = pqueue_pop(s->d1->buffered_app_data.q)) != NULL) {
-        rdata = (DTLS1_RECORD_DATA *)item->data;
-        if (rdata->rbuf.buf) {
-            OPENSSL_free(rdata->rbuf.buf);
-        }
-        OPENSSL_free(item->data);
-        pitem_free(item);
-    }
 }
 
+
 void dtls1_free(SSL *s)
 {
     ssl3_free(s);
@@ -456,7 +471,7 @@ void dtls1_stop_timer(SSL *s)
     BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT, 0,
              &(s->d1->next_timeout));
     /* Clear retransmission buffer */
-    dtls1_clear_record_buffer(s);
+    dtls1_clear_sent_buffer(s);
 }
 
 int dtls1_check_timeout_num(SSL *s)

diff --git a/ssl/d1_srvr.c b/ssl/d1_srvr.c
index e677d880f0ac927442f93bd6a14af85b522d7c53..bc875b53c9a0d9b4d97a83897d715cafe97ef2d1 100644 (file)
--- a/ssl/d1_srvr.c
+++ b/ssl/d1_srvr.c
@@ -313,7 +313,7 @@ int dtls1_accept(SSL *s)
         case SSL3_ST_SW_HELLO_REQ_B:
 
             s->shutdown = 0;
-            dtls1_clear_record_buffer(s);
+            dtls1_clear_sent_buffer(s);
             dtls1_start_timer(s);
             ret = ssl3_send_hello_request(s);
             if (ret <= 0)
@@ -894,6 +894,7 @@ int dtls1_accept(SSL *s)
             /* next message is server hello */
             s->d1->handshake_write_seq = 0;
             s->d1->next_handshake_write_seq = 0;
+            dtls1_clear_received_buffer(s);
             goto end;
             /* break; */
 

diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index 3dd2a54b22ac8d818aea6d2315661390b6e6197f..e358031db41459d08afc43a8f60b71c37d8dff17 100644 (file)
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -1248,7 +1248,8 @@ int dtls1_retransmit_message(SSL *s, unsigned short seq,
                              unsigned long frag_off, int *found);
 int dtls1_get_queue_priority(unsigned short seq, int is_ccs);
 int dtls1_retransmit_buffered_messages(SSL *s);
-void dtls1_clear_record_buffer(SSL *s);
+void dtls1_clear_received_buffer(SSL *s);
+void dtls1_clear_sent_buffer(SSL *s);
 void dtls1_get_message_header(unsigned char *data,
                               struct hm_header_st *msg_hdr);
 void dtls1_get_ccs_header(unsigned char *data, struct ccs_header_st *ccs_hdr);