CAPI engine: add support for RSA_NO_PADDING

Since the SSL code started using RSA_NO_PADDING, the CAPI engine became
unusable.  This change fixes that.

Fixes #7131

Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7132)

diff --git a/engines/e_capi.c b/engines/e_capi.c
index 8c08872bfdf446ec8ef0c01733d8fa9612a9c239..814a3253f2bf8959ff47cffdbe6cd1f2ccb68744 100644 (file)
--- a/engines/e_capi.c
+++ b/engines/e_capi.c
@@ -900,6 +900,8 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from,
     unsigned char *tmpbuf;
     CAPI_KEY *capi_key;
     CAPI_CTX *ctx;
+    DWORD flags = 0;
+
     ctx = ENGINE_get_ex_data(rsa->engine, capi_idx);
 
     CAPI_trace(ctx, "Called capi_rsa_priv_dec()\n");
@@ -910,12 +912,23 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from,
         return -1;
     }
 
-    if (padding != RSA_PKCS1_PADDING) {
-        char errstr[10];
-        BIO_snprintf(errstr, 10, "%d", padding);
-        CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_UNSUPPORTED_PADDING);
-        ERR_add_error_data(2, "padding=", errstr);
-        return -1;
+    switch (padding) {
+    case RSA_PKCS1_PADDING:
+        /* Nothing to do */
+        break;
+#ifdef CRYPT_DECRYPT_RSA_NO_PADDING_CHECK
+    case RSA_NO_PADDING:
+        flags = CRYPT_DECRYPT_RSA_NO_PADDING_CHECK;
+        break;
+#endif
+    default:
+        {
+            char errstr[10];
+            BIO_snprintf(errstr, 10, "%d", padding);
+            CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_UNSUPPORTED_PADDING);
+            ERR_add_error_data(2, "padding=", errstr);
+            return -1;
+        }
     }
 
     /* Create temp reverse order version of input */
@@ -927,14 +940,17 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from,
         tmpbuf[flen - i - 1] = from[i];
 
     /* Finally decrypt it */
-    if (!CryptDecrypt(capi_key->key, 0, TRUE, 0, tmpbuf, &flen)) {
+    if (!CryptDecrypt(capi_key->key, 0, TRUE, flags, tmpbuf, &flen)) {
         CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_DECRYPT_ERROR);
         capi_addlasterror();
+        OPENSSL_cleanse(tmpbuf, flen);
         OPENSSL_free(tmpbuf);
         return -1;
-    } else
+    } else {
         memcpy(to, tmpbuf, flen);
+    }
 
+    OPENSSL_cleanse(tmpbuf, flen);
     OPENSSL_free(tmpbuf);
 
     return flen;