trace: don't leak the line prefix

The openssl app registers trace callbacks which automatically
set a line prefix in the OSSL_TRACE_CTRL_BEGIN callback.
This prefix needs to be cleared in the OSSL_TRACE_CTRL_END
callback, otherwise a memory leak is reported when openssl
is built with crypto-mdebug enabled.

This leak causes the tests to fail when tracing and memory
debugging are enabled.

The leak can be observed by any command that produces trace
output, e.g. by

  OPENSSL_TRACE=ANY util/shlib_wrap.sh  apps/openssl version
  ...
  [00:19:14]  4061 file=apps/bf_prefix.c, line=152, ...
  26 bytes leaked in 1 chunks

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8463)

diff --git a/apps/openssl.c b/apps/openssl.c
index 72a0bb008550c3f4a45ee70d0d26fbb88ef681b8..db1dbb767d1f5519f1c87c48628120b426b28535 100644 (file)
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -126,30 +126,17 @@ typedef struct tracedata_st {
 static size_t internal_trace_cb(const char *buf, size_t cnt,
                                 int category, int cmd, void *vdata)
 {
-    int ret;
+    int ret = 0;
     tracedata *trace_data = vdata;
-    int set_prefix = 0;
+    union {
+        CRYPTO_THREAD_ID tid;
+        unsigned long ltid;
+    } tid;
+    char buffer[256];
 
     switch (cmd) {
     case OSSL_TRACE_CTRL_BEGIN:
         trace_data->ingroup = 1;
-        set_prefix = 1;
-        break;
-    case OSSL_TRACE_CTRL_DURING:
-        if (!trace_data->ingroup)
-            set_prefix = 1;
-        break;
-    case OSSL_TRACE_CTRL_END:
-        trace_data->ingroup = 0;
-        break;
-    }
-
-    if (set_prefix) {
-        union {
-            CRYPTO_THREAD_ID tid;
-            unsigned long ltid;
-        } tid;
-        char buffer[256];
 
         tid.ltid = 0;
         tid.tid = CRYPTO_THREAD_get_current_id();
@@ -158,8 +145,17 @@ static size_t internal_trace_cb(const char *buf, size_t cnt,
                      OSSL_trace_get_category_name(category));
         BIO_ctrl(trace_data->bio, PREFIX_CTRL_SET_PREFIX,
                  strlen(buffer), buffer);
+        break;
+    case OSSL_TRACE_CTRL_WRITE:
+        ret = BIO_write(trace_data->bio, buf, cnt);
+        break;
+    case OSSL_TRACE_CTRL_END:
+        trace_data->ingroup = 0;
+
+        BIO_ctrl(trace_data->bio, PREFIX_CTRL_SET_PREFIX, 0, NULL);
+
+        break;
     }
-    ret = BIO_write(trace_data->bio, buf, cnt);
 
     return ret < 0 ? 0 : ret;
 }

diff --git a/crypto/trace.c b/crypto/trace.c
index 4e93c43defb51139f8ed79b53d6cfe92390c664b..533770c669f646070fe3cdeb54112cf83921c325 100644 (file)
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -65,7 +65,7 @@ static int trace_write(BIO *channel,
                        const char *buf, size_t num, size_t *written)
 {
     struct trace_data_st *ctx = BIO_get_data(channel);
-    size_t cnt = ctx->callback(buf, num, ctx->category, OSSL_TRACE_CTRL_DURING,
+    size_t cnt = ctx->callback(buf, num, ctx->category, OSSL_TRACE_CTRL_WRITE,
                                ctx->data);
 
     *written = cnt;

diff --git a/include/openssl/trace.h b/include/openssl/trace.h
index da0ba0b5dfa2ac10ac9b0dd8855990bad3b6d949..767b19fa7d7664909be8ca578582823938f86ac8 100644 (file)
--- a/include/openssl/trace.h
+++ b/include/openssl/trace.h
@@ -95,7 +95,7 @@ typedef size_t (*OSSL_trace_cb)(const char *buffer, size_t count,
  * Possible |cmd| numbers.
  */
 # define OSSL_TRACE_CTRL_BEGIN  0
-# define OSSL_TRACE_CTRL_DURING 1
+# define OSSL_TRACE_CTRL_WRITE  1
 # define OSSL_TRACE_CTRL_END    2
 
 /*