zero pad DHE public key in ServerKeyExchange message for interop

[oweals/openssl.git] / ssl / s3_srvr.c

diff --git a/ssl/s3_srvr.c b/ssl/s3_srvr.c
index 0c43c493ed6e2b875d3bb0404136a4cda24249bd..0e57cb357098db38fefac12f05c4ea23a7621794 100644 (file)
--- a/ssl/s3_srvr.c
+++ b/ssl/s3_srvr.c
@@ -980,7 +980,8 @@ int ssl3_get_client_hello(SSL *s)
 
         session_length = *(p + SSL3_RANDOM_SIZE);
 
-        if (SSL3_RANDOM_SIZE + session_length + 1 >= (d + n) - p) {
+        if (SSL3_RANDOM_SIZE + session_length + 1
+                >= (unsigned int)((d + n) - p)) {
             al = SSL_AD_DECODE_ERROR;
             SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
             goto f_err;
@@ -1061,7 +1062,7 @@ int ssl3_get_client_hello(SSL *s)
         }
         cookie_len = *(p++);
 
-        if ((d + n ) - p < cookie_len) {
+        if ((unsigned int)((d + n ) - p) < cookie_len) {
             al = SSL_AD_DECODE_ERROR;
             SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
             goto f_err;
@@ -1464,9 +1465,9 @@ int ssl3_get_client_hello(SSL *s)
 
     /* Handles TLS extensions that we couldn't check earlier */
     if (s->version >= SSL3_VERSION) {
-        if (ssl_check_clienthello_tlsext_late(s) <= 0) {
+        if (!ssl_check_clienthello_tlsext_late(s, &al)) {
             SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_CLIENTHELLO_TLSEXT);
-            goto err;
+            goto f_err;
         }
     }
 
@@ -1600,6 +1601,9 @@ int ssl3_send_server_key_exchange(SSL *s)
     unsigned int u;
 #endif
 #ifndef OPENSSL_NO_DH
+# ifdef OPENSSL_NO_RSA
+    int j;
+# endif
     DH *dh = NULL, *dhp;
 #endif
 #ifndef OPENSSL_NO_ECDH
@@ -1860,6 +1864,16 @@ int ssl3_send_server_key_exchange(SSL *s)
             if ((i == 2) && (type & SSL_kSRP))
                 n += 1 + nr[i];
             else
+#endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime, so use the length of the prime here
+             */
+            if ((i == 2) && (type & (SSL_kEDH)))
+                n += 2 + nr[0];
+            else
 #endif
                 n += 2 + nr[i];
         }
@@ -1872,6 +1886,11 @@ int ssl3_send_server_key_exchange(SSL *s)
                 goto f_err;
             }
             kn = EVP_PKEY_size(pkey);
+            /* Allow space for signature algorithm */
+            if (SSL_USE_SIGALGS(s))
+                kn += 2;
+            /* Allow space for signature length */
+            kn += 2;
         } else {
             pkey = NULL;
             kn = 0;
@@ -1889,6 +1908,20 @@ int ssl3_send_server_key_exchange(SSL *s)
                 *p = nr[i];
                 p++;
             } else
+#endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime
+             */
+            if ((i == 2) && (type & (SSL_kEDH))) {
+                s2n(nr[0], p);
+                for (j = 0; j < (nr[0] - nr[2]); ++j) {
+                    *p = 0;
+                    ++p;
+                }
+            } else
 #endif
                 s2n(nr[i], p);
             BN_bn2bin(r[i], p);
@@ -2320,7 +2353,8 @@ int ssl3_get_client_key_exchange(SSL *s)
             if (!(s->options & SSL_OP_SSLEAY_080_CLIENT_DH_BUG)) {
                 SSLerr(SSL_F_SSL3_GET_CLIENT_KEY_EXCHANGE,
                        SSL_R_DH_PUBLIC_VALUE_LENGTH_IS_WRONG);
-                goto err;
+                al = SSL_AD_HANDSHAKE_FAILURE;
+                goto f_err;
             } else {
                 p -= 2;
                 i = (int)n;
@@ -2373,9 +2407,10 @@ int ssl3_get_client_key_exchange(SSL *s)
         i = DH_compute_key(p, pub, dh_srvr);
 
         if (i <= 0) {
+            al = SSL_AD_HANDSHAKE_FAILURE;
             SSLerr(SSL_F_SSL3_GET_CLIENT_KEY_EXCHANGE, ERR_R_DH_LIB);
             BN_clear_free(pub);
-            goto err;
+            goto f_err;
         }
 
         DH_free(s->s3->tmp.dh);
@@ -2673,12 +2708,14 @@ int ssl3_get_client_key_exchange(SSL *s)
             i = *p;
             p += 1;
             if (n != 1 + i) {
-                SSLerr(SSL_F_SSL3_GET_CLIENT_KEY_EXCHANGE, ERR_R_EC_LIB);
-                goto err;
+                SSLerr(SSL_F_SSL3_GET_CLIENT_KEY_EXCHANGE, SSL_R_LENGTH_MISMATCH);
+                al = SSL_AD_DECODE_ERROR;
+                goto f_err;
             }
             if (EC_POINT_oct2point(group, clnt_ecpoint, p, i, bn_ctx) == 0) {
                 SSLerr(SSL_F_SSL3_GET_CLIENT_KEY_EXCHANGE, ERR_R_EC_LIB);
-                goto err;
+                al = SSL_AD_HANDSHAKE_FAILURE;
+                goto f_err;
             }
             /*
              * p is pointing to somewhere in the buffer currently, so set it
@@ -3210,6 +3247,12 @@ int ssl3_get_client_certificate(SSL *s)
         goto f_err;
     }
     for (nc = 0; nc < llen;) {
+        if (nc + 3 > llen) {
+            al = SSL_AD_DECODE_ERROR;
+            SSLerr(SSL_F_SSL3_GET_CLIENT_CERTIFICATE,
+                   SSL_R_CERT_LENGTH_MISMATCH);
+            goto f_err;
+        }
         n2l3(p, l);
         if ((l + nc + 3) > llen) {
             al = SSL_AD_DECODE_ERROR;
@@ -3494,37 +3537,34 @@ int ssl3_send_cert_status(SSL *s)
 {
     if (s->state == SSL3_ST_SW_CERT_STATUS_A) {
         unsigned char *p;
+        size_t msglen;
+
         /*-
          * Grow buffer if need be: the length calculation is as
-         * follows 1 (message type) + 3 (message length) +
+         * follows handshake_header_length +
          * 1 (ocsp response type) + 3 (ocsp response length)
          * + (ocsp response)
          */
-        if (!BUF_MEM_grow(s->init_buf, 8 + s->tlsext_ocsp_resplen)) {
+        msglen = 4 + s->tlsext_ocsp_resplen;
+        if (!BUF_MEM_grow(s->init_buf, SSL_HM_HEADER_LENGTH(s) + msglen)) {
             s->state = SSL_ST_ERR;
             return -1;
         }
 
-        p = (unsigned char *)s->init_buf->data;
+        p = ssl_handshake_start(s);
 
-        /* do the header */
-        *(p++) = SSL3_MT_CERTIFICATE_STATUS;
-        /* message length */
-        l2n3(s->tlsext_ocsp_resplen + 4, p);
         /* status type */
         *(p++) = s->tlsext_status_type;
         /* length of OCSP response */
         l2n3(s->tlsext_ocsp_resplen, p);
         /* actual response */
         memcpy(p, s->tlsext_ocsp_resp, s->tlsext_ocsp_resplen);
-        /* number of bytes to write */
-        s->init_num = 8 + s->tlsext_ocsp_resplen;
-        s->state = SSL3_ST_SW_CERT_STATUS_B;
-        s->init_off = 0;
+
+        ssl_set_handshake_header(s, SSL3_MT_CERTIFICATE_STATUS, msglen);
     }
 
     /* SSL3_ST_SW_CERT_STATUS_B */
-    return (ssl3_do_write(s, SSL3_RT_HANDSHAKE));
+    return (ssl_do_write(s));
 }
 
 # ifndef OPENSSL_NO_NEXTPROTONEG