Honour mandatory digest on private key in has_usable_cert()

If the private key says it can only support one specific digest, then
don't ask it to perform a different one.

Fixes: #7348

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7408)

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index b8b9fbda3900cb67da0d206db34279e98523a831..2e785a909b9006f73197afd637418da5006ed38e 100644 (file)
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2496,7 +2496,8 @@ static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu)
 static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
 {
     const SIGALG_LOOKUP *lu;
-    int mdnid, pknid;
+    int mdnid, pknid, default_mdnid;
+    int mandatory_md = 0;
     size_t i;
 
     /* TLS 1.2 callers can override lu->sig_idx, but not TLS 1.3 callers. */
@@ -2504,12 +2505,26 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         idx = sig->sig_idx;
     if (!ssl_has_cert(s, idx))
         return 0;
+    /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */
+    ERR_set_mark();
+    switch (EVP_PKEY_get_default_digest_nid(s->cert->pkeys[idx].privatekey,
+                                            &default_mdnid)) {
+    case 2:
+        mandatory_md = 1;
+        break;
+    case 1:
+        break;
+    default: /* If it didn't report a mandatory NID, for whatever reasons,
+              * just clear the error and allow all hashes to be used. */
+        ERR_pop_to_mark();
+    }
     if (s->s3->tmp.peer_cert_sigalgs != NULL) {
         for (i = 0; i < s->s3->tmp.peer_cert_sigalgslen; i++) {
             lu = tls1_lookup_sigalg(s->s3->tmp.peer_cert_sigalgs[i]);
             if (lu == NULL
                 || !X509_get_signature_info(s->cert->pkeys[idx].x509, &mdnid,
-                                            &pknid, NULL, NULL))
+                                            &pknid, NULL, NULL)
+                || (mandatory_md && mdnid != default_mdnid))
                 continue;
             /*
              * TODO this does not differentiate between the
@@ -2522,7 +2537,7 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         }
         return 0;
     }
-    return 1;
+    return !mandatory_md || sig->hash == default_mdnid;
 }
 
 /*