Fix a crash or unbounded allocation in RSA_padding_add_PKCS1_PSS_mgf1
authorBernd Edlinger <bernd.edlinger@hotmail.de>
Wed, 8 Mar 2017 17:53:36 +0000 (18:53 +0100)
committerAndy Polyakov <appro@openssl.org>
Mon, 13 Mar 2017 21:11:36 +0000 (22:11 +0100)
and RSA_verify_PKCS1_PSS_mgf1 with 512-bit RSA vs. sha-512.

Reviewed-by: Andy Polyakov <appro@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2881)

(cherry picked from commit e653b6cd74f70c04b0b96b07df00680b427603af)

crypto/rsa/rsa_pss.c

index 41bc0844e48ed93c46ea9f9f1d81b380ca3c4779..2c3fd73b0996a90eeaa36a76496b265423b2e16b 100644 (file)
@@ -122,7 +122,11 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         EM++;
         emLen--;
     }
-    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
+    if (emLen < hLen + 2) {
+        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
+        goto err;
+    }
+    if (sLen > emLen - hLen - 2) { /* sLen can be small negative */
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
         goto err;
     }
@@ -222,9 +226,14 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         *EM++ = 0;
         emLen--;
     }
+    if (emLen < hLen + 2) {
+        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,
+               RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
+        goto err;
+    }
     if (sLen == -2) {
         sLen = emLen - hLen - 2;
-    } else if (emLen < (hLen + sLen + 2)) {
+    } else if (sLen > emLen - hLen - 2) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,
                RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
         goto err;