TLSv1.3: additional checks in SSL_set_record_padding_callback

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11589)

diff --git a/doc/man3/SSL_CTX_set_record_padding_callback.pod b/doc/man3/SSL_CTX_set_record_padding_callback.pod
index 3df66218fe1db874fc038ea588e866f5ebc629bc..445953ef073988967b1e070715ed578bd0bcb411 100644 (file)
--- a/doc/man3/SSL_CTX_set_record_padding_callback.pod
+++ b/doc/man3/SSL_CTX_set_record_padding_callback.pod
@@ -16,7 +16,7 @@ SSL_set_block_padding - install callback to specify TLS 1.3 record padding
  #include <openssl/ssl.h>
 
  void SSL_CTX_set_record_padding_callback(SSL_CTX *ctx, size_t (*cb)(SSL *s, int type, size_t len, void *arg));
- void SSL_set_record_padding_callback(SSL *ssl, size_t (*cb)(SSL *s, int type, size_t len, void *arg));
+ int SSL_set_record_padding_callback(SSL *ssl, size_t (*cb)(SSL *s, int type, size_t len, void *arg));
 
  void SSL_CTX_set_record_padding_callback_arg(SSL_CTX *ctx, void *arg);
  void *SSL_CTX_get_record_padding_callback_arg(const SSL_CTX *ctx);
@@ -32,6 +32,8 @@ SSL_set_block_padding - install callback to specify TLS 1.3 record padding
 SSL_CTX_set_record_padding_callback() or SSL_set_record_padding_callback()
 can be used to assign a callback function I<cb> to specify the padding
 for TLS 1.3 records. The value set in B<ctx> is copied to a new SSL by SSL_new().
+Kernel TLS is not possible if the record padding callback is set, and the callback
+function cannot be set if Kernel TLS is already configured for the current SSL object.
 
 SSL_CTX_set_record_padding_callback_arg() and SSL_set_record_padding_callback_arg()
 assign a value B<arg> that is passed to the callback when it is invoked. The value
@@ -64,6 +66,9 @@ indicates no padding will be added. A return value that causes the record to
 exceed the maximum record size (SSL3_RT_MAX_PLAIN_LENGTH) will pad out to the
 maximum record size.
 
+The SSL_CTX_get_record_padding_callback_arg() function returns 1 on success or 0 if
+the callback function is not set because Kernel TLS is configured for the SSL object.
+
 =head1 NOTES
 
 The default behavior is to add no padding to the record.
@@ -84,6 +89,9 @@ L<ssl(7)>, L<SSL_new(3)>
 
 The record padding API was added for TLS 1.3 support in OpenSSL 1.1.1.
 
+The return type of SSL_CTX_set_record_padding_callback() function was
+changed to int in OpenSSL 3.0.
+
 =head1 COPYRIGHT
 
 Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.

diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index c039b2a04a93c9d544afcec692e49ccff1afd0d2..f855f9470d5e5eefe9cc846c854a9675e8e58e7f 100644 (file)
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -2167,7 +2167,7 @@ void SSL_CTX_set_record_padding_callback_arg(SSL_CTX *ctx, void *arg);
 void *SSL_CTX_get_record_padding_callback_arg(const SSL_CTX *ctx);
 int SSL_CTX_set_block_padding(SSL_CTX *ctx, size_t block_size);
 
-void SSL_set_record_padding_callback(SSL *ssl,
+int SSL_set_record_padding_callback(SSL *ssl,
                                     size_t (*cb) (SSL *ssl, int type,
                                                   size_t len, void *arg));
 void SSL_set_record_padding_callback_arg(SSL *ssl, void *arg);

diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index c2c238871662fb94787e6dacd20aad1179f8e0a3..9fb65b68258600fe6530853e180e7d8b4857389e 100644 (file)
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -4645,11 +4645,18 @@ int SSL_CTX_set_block_padding(SSL_CTX *ctx, size_t block_size)
     return 1;
 }
 
-void SSL_set_record_padding_callback(SSL *ssl,
+int SSL_set_record_padding_callback(SSL *ssl,
                                      size_t (*cb) (SSL *ssl, int type,
                                                    size_t len, void *arg))
 {
-    ssl->record_padding_cb = cb;
+    BIO *b;
+
+    b = SSL_get_wbio(ssl);
+    if (b == NULL || !BIO_get_ktls_send(b)) {
+        ssl->record_padding_cb = cb;
+        return 1;
+    }
+    return 0;
 }
 
 void SSL_set_record_padding_callback_arg(SSL *ssl, void *arg)