Don't allow PKCS#7/CMS encrypt with PSS.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index b091746b1c83433178a1a3a85aaaa9eb464d8ed0..c030c275602ff976b99734f9bebadeded28843c5 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -413,6 +413,8 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         break;
 
     case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
+        if (pkey_is_pss(pkey))
+            return -2;
         if (arg1 == 0)
             PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
         break;
@@ -425,6 +427,8 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         break;
 
     case ASN1_PKEY_CTRL_CMS_ENVELOPE:
+        if (pkey_is_pss(pkey))
+            return -2;
         if (arg1 == 0)
             return rsa_cms_encrypt(arg2);
         else if (arg1 == 1)
@@ -432,6 +436,8 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         break;
 
     case ASN1_PKEY_CTRL_CMS_RI_TYPE:
+        if (pkey_is_pss(pkey))
+            return -2;
         *(int *)arg2 = CMS_RECIPINFO_TRANS;
         return 1;
 #endif