Don't attempt to send fragments > max_send_fragment in DTLS

We were allocating the write buffer based on the size of max_send_fragment,
but ignoring it when writing data. We should fragment handshake messages
if they exceed max_send_fragment and reject application data writes that
are too large.

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3287)

diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index 8d75d53eca9b8eda8ca2ecf93c46183405bfcfba..f7ea73668e00d98257ad94c469b520b48ed07503 100644 (file)
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -2346,6 +2346,7 @@ int ERR_load_SSL_strings(void);
 # define SSL_R_ENCRYPTED_LENGTH_TOO_LONG                  150
 # define SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST              151
 # define SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN             204
+# define SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE                  194
 # define SSL_R_EXCESSIVE_MESSAGE_SIZE                     152
 # define SSL_R_EXTRA_DATA_IN_MESSAGE                      153
 # define SSL_R_FAILED_TO_INIT_ASYNC                       405

diff --git a/ssl/record/rec_layer_d1.c b/ssl/record/rec_layer_d1.c
index 5c9a18082ac27b150f6575360d5893c6237781f9..1686edd7b3fe1926149b51764c38f88e365ae35b 100644 (file)
--- a/ssl/record/rec_layer_d1.c
+++ b/ssl/record/rec_layer_d1.c
@@ -988,6 +988,11 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf,
     if (len == 0 && !create_empty_fragment)
         return 0;
 
+    if (len > s->max_send_fragment) {
+        SSLerr(SSL_F_DO_DTLS1_WRITE, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE);
+        return 0;
+    }
+
     sess = s->session;
 
     if ((sess == NULL) ||

diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index 73e0ae15c1a6c7198185e304277c451c4940da3d..be4c0c00c1eb5e36c295cf93a90989f5ab38e4ea 100644 (file)
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -415,6 +415,8 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "error in received cipher list"},
     {ERR_REASON(SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN),
      "error setting tlsa base domain"},
+    {ERR_REASON(SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE),
+     "exceeds max fragment size"},
     {ERR_REASON(SSL_R_EXCESSIVE_MESSAGE_SIZE), "excessive message size"},
     {ERR_REASON(SSL_R_EXTRA_DATA_IN_MESSAGE), "extra data in message"},
     {ERR_REASON(SSL_R_FAILED_TO_INIT_ASYNC), "failed to init async"},

diff --git a/ssl/statem/statem_dtls.c b/ssl/statem/statem_dtls.c
index 043f41b724f07b10e10271c791e427deb45d793a..37e7fea8abc048a2b8b6427eccc26798fa8884ee 100644 (file)
--- a/ssl/statem/statem_dtls.c
+++ b/ssl/statem/statem_dtls.c
@@ -214,9 +214,8 @@ int dtls1_do_write(SSL *s, int type)
         else
             len = s->init_num;
 
-        /* Shouldn't ever happen */
-        if (len > INT_MAX)
-            len = INT_MAX;
+        if (len > s->max_send_fragment)
+            len = s->max_send_fragment;
 
         /*
          * XDTLS: this function is too long.  split out the CCS part