Don't attempt to send fragments > max_send_fragment in DTLS

We were allocating the write buffer based on the size of max_send_fragment,
but ignoring it when writing data. We should fragment handshake messages
if they exceed max_send_fragment and reject application data writes that
are too large.

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3286)

diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index 6e3b9c52e5291c19fd36008ac50cdefecb4c1757..c14859fb8341058ef299388cd30aa11f9d324567 100644 (file)
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -2607,6 +2607,7 @@ int ERR_load_SSL_strings(void);
 # define SSL_R_ENCRYPTED_LENGTH_TOO_LONG                  150
 # define SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST              151
 # define SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN             204
+# define SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE                  194
 # define SSL_R_EXCESSIVE_MESSAGE_SIZE                     152
 # define SSL_R_EXTRA_DATA_IN_MESSAGE                      153
 # define SSL_R_EXT_LENGTH_MISMATCH                        163

diff --git a/ssl/record/rec_layer_d1.c b/ssl/record/rec_layer_d1.c
index ca7f42737711e22725b8ac69ed5340fca1edf686..243eff70040dbaf06c3504ede54a7e079ad957ee 100644 (file)
--- a/ssl/record/rec_layer_d1.c
+++ b/ssl/record/rec_layer_d1.c
@@ -882,6 +882,11 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf,
     if (len == 0 && !create_empty_fragment)
         return 0;
 
+    if (len > s->max_send_fragment) {
+        SSLerr(SSL_F_DO_DTLS1_WRITE, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE);
+        return 0;
+    }
+
     sess = s->session;
 
     if ((sess == NULL) ||

diff --git a/ssl/ssl_err.c b/ssl/ssl_err.c
index c7e407fc2779642317fec649120052f851689774..296ce0de03556f0bf92a4309b33c2824b9fbcf5a 100644 (file)
--- a/ssl/ssl_err.c
+++ b/ssl/ssl_err.c
@@ -590,6 +590,8 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "error in received cipher list"},
     {ERR_REASON(SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN),
      "error setting tlsa base domain"},
+    {ERR_REASON(SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE),
+     "exceeds max fragment size"},
     {ERR_REASON(SSL_R_EXCESSIVE_MESSAGE_SIZE), "excessive message size"},
     {ERR_REASON(SSL_R_EXTRA_DATA_IN_MESSAGE), "extra data in message"},
     {ERR_REASON(SSL_R_EXT_LENGTH_MISMATCH), "ext length mismatch"},

diff --git a/ssl/statem/statem_dtls.c b/ssl/statem/statem_dtls.c
index 34964dbd5d791618a7c080c0c0fcd5174dcdf89c..b2ba35763a09b7b2d360f4ca4ad97fa5c1f0216b 100644 (file)
--- a/ssl/statem/statem_dtls.c
+++ b/ssl/statem/statem_dtls.c
@@ -214,6 +214,9 @@ int dtls1_do_write(SSL *s, int type)
         else
             len = s->init_num;
 
+        if (len > s->max_send_fragment)
+            len = s->max_send_fragment;
+
         /*
          * XDTLS: this function is too long.  split out the CCS part
          */