trace: fix out-of-bound memory access

When OSSL_trace_get_category_num() is called with an unknown category
name, it returns -1. This case needs to be considered in order to
avoid out-of-bound memory access to the `trace_channels` array.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8552)

diff --git a/crypto/trace.c b/crypto/trace.c
index 6299a688be54ce5f9306d17a4e0cc9d13ec7171f..5e2fec299ac9b05450489391396b003d1e461cbb 100644 (file)
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -431,7 +431,8 @@ int OSSL_trace_enabled(int category)
     int ret = 0;
 #ifndef OPENSSL_NO_TRACE
     category = ossl_trace_get_category(category);
-    ret = trace_channels[category].bio != NULL;
+    if (category >= 0)
+        ret = trace_channels[category].bio != NULL;
 #endif
     return ret;
 }
@@ -443,6 +444,9 @@ BIO *OSSL_trace_begin(int category)
     char *prefix = NULL;
 
     category = ossl_trace_get_category(category);
+    if (category < 0)
+        return NULL;
+
     channel = trace_channels[category].bio;
     prefix = trace_channels[category].prefix;