Avoid creating invalid rsa pss params

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8621)

(cherry picked from commit 491360e7ab2f09fdaadfcd9ff84c425c8f4e5b03)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 75debb3e0a9d2a3ff78623b9332b2950e7d4c973..637b02c214d2cfe413ab46a9c483b1f0874c5d7d 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -583,10 +583,12 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         return NULL;
     if (saltlen == -1) {
         saltlen = EVP_MD_size(sigmd);
-    } else if (saltlen == -2) {
+    } else if (saltlen == -2 || saltlen == -3) {
         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
         if ((EVP_PKEY_bits(pk) & 0x7) == 1)
             saltlen--;
+        if (saltlen < 0)
+            return NULL;
     }
 
     return rsa_pss_params_create(sigmd, mgf1md, saltlen);

diff --git a/test/recipes/80-test_cms.t b/test/recipes/80-test_cms.t
index f038bea31d5b774ed55ef220ad1f814032c4a39e..52b822ef6fde4dbc51070328a5a0f44d357db8c4 100644 (file)
--- a/test/recipes/80-test_cms.t
+++ b/test/recipes/80-test_cms.t
@@ -308,6 +308,14 @@ my @smime_cms_param_tests = (
        "-CAfile", catfile($smdir, "smroot.pem"), "-out", "smtst.txt" ]
     ],
 
+    [ "signed content test streaming PEM format, RSA keys, PSS signature, saltlen=-3",
+      [ "-sign", "-in", $smcont, "-outform", "PEM", "-nodetach",
+       "-signer", catfile($smdir, "smrsa1.pem"), "-keyopt", "rsa_padding_mode:pss",
+       "-keyopt", "rsa_pss_saltlen:-3", "-out", "test.cms" ],
+      [ "-verify", "-in", "test.cms", "-inform", "PEM",
+       "-CAfile", catfile($smdir, "smroot.pem"), "-out", "smtst.txt" ]
+    ],
+
     [ "signed content test streaming PEM format, RSA keys, PSS signature, no attributes",
       [ "-sign", "-in", $smcont, "-outform", "PEM", "-nodetach", "-noattr",
        "-signer", catfile($smdir, "smrsa1.pem"), "-keyopt", "rsa_padding_mode:pss",