zero pad DHE public key in ServerKeyExchange message for interop

Some versions of the Microsoft TLS stack have problems when the DHE public key
is encoded with fewer bytes than the DHE prime. (Backported from master)

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1350)

diff --git a/ssl/s3_srvr.c b/ssl/s3_srvr.c
index aa591eb0654113379d6d914161ab084bb88f8337..0e57cb357098db38fefac12f05c4ea23a7621794 100644 (file)
--- a/ssl/s3_srvr.c
+++ b/ssl/s3_srvr.c
@@ -1601,6 +1601,9 @@ int ssl3_send_server_key_exchange(SSL *s)
     unsigned int u;
 #endif
 #ifndef OPENSSL_NO_DH
+# ifdef OPENSSL_NO_RSA
+    int j;
+# endif
     DH *dh = NULL, *dhp;
 #endif
 #ifndef OPENSSL_NO_ECDH
@@ -1861,6 +1864,16 @@ int ssl3_send_server_key_exchange(SSL *s)
             if ((i == 2) && (type & SSL_kSRP))
                 n += 1 + nr[i];
             else
+#endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime, so use the length of the prime here
+             */
+            if ((i == 2) && (type & (SSL_kEDH)))
+                n += 2 + nr[0];
+            else
 #endif
                 n += 2 + nr[i];
         }
@@ -1895,6 +1908,20 @@ int ssl3_send_server_key_exchange(SSL *s)
                 *p = nr[i];
                 p++;
             } else
+#endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime
+             */
+            if ((i == 2) && (type & (SSL_kEDH))) {
+                s2n(nr[0], p);
+                for (j = 0; j < (nr[0] - nr[2]); ++j) {
+                    *p = 0;
+                    ++p;
+                }
+            } else
 #endif
                 s2n(nr[i], p);
             BN_bn2bin(r[i], p);