EVP: Don't call digest_custom() quite so early

A huge problem with calling digest_custom() already in the
initialization of DigestSign, DigestVerify etc, is that it force all
callers to know that certain controls must be performed before Init
and the rest after.  This has lead to quite interesting hacks in our
own openssl app, where the SM2 ID had to get special treatment instead
of just being another sign option or verification option among others.

This change moves the call of digest_custom() to the Update and Final
functions, to be done exactly once, subject to a flag that's set in
the Init function.  Seeing to the process of data, through these
operations, this makes no difference at all.  Seeing to making it
possible to perform all controls after the Init call, this makes a
huge difference.

Fixes #11293

Reviewed-by: Paul Yang <kaishen.yy@antfin.com>
(Merged from https://github.com/openssl/openssl/pull/11302)

diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index 4b2cb4eb35f14c15aa4b5d294351fc831531686f..1948f234cab8325d9351d3b90baa8b02f7e01903 100644 (file)
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -250,8 +250,9 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
      * This indicates the current algorithm requires
      * special treatment before hashing the tbs-message.
      */
+    ctx->pctx->flag_call_digest_custom = 0;
     if (ctx->pctx->pmeth->digest_custom != NULL)
-        return ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx);
+        ctx->pctx->flag_call_digest_custom = 1;
 
     return 1;
 }
@@ -301,6 +302,12 @@ int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
                                                       data, dsize);
 
  legacy:
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     return EVP_DigestUpdate(ctx, data, dsize);
 }
 
@@ -323,6 +330,12 @@ int EVP_DigestVerifyUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
                                                         data, dsize);
 
  legacy:
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     return EVP_DigestUpdate(ctx, data, dsize);
 }
 
@@ -348,6 +361,12 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
         return 0;
     }
 
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     if (pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM) {
         if (sigret == NULL)
             return pctx->pmeth->signctx(pctx, sigret, siglen, ctx);
@@ -458,6 +477,12 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig,
         return 0;
     }
 
+    /* do_sigver_init() checked that |digest_custom| is non-NULL */
+    if (pctx->flag_call_digest_custom
+        && !ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx))
+        return 0;
+    pctx->flag_call_digest_custom = 0;
+
     if (pctx->pmeth->verifyctx != NULL)
         vctx = 1;
     else

diff --git a/include/crypto/evp.h b/include/crypto/evp.h
index 744731aefe2dc8aed9f9a5353c8316207b844b2e..c9d3075b82754bb726a400d4e706e336ba360d2e 100644 (file)
--- a/include/crypto/evp.h
+++ b/include/crypto/evp.h
@@ -74,6 +74,8 @@ struct evp_pkey_ctx_st {
     EVP_PKEY *peerkey;
     /* Algorithm specific data */
     void *data;
+    /* Indicator if digest_custom needs to be called */
+    unsigned int flag_call_digest_custom:1;
 } /* EVP_PKEY_CTX */ ;
 
 #define EVP_PKEY_FLAG_DYNAMIC   1