This patch adds support for the Linux TLS Tx socket option.
If the socket option is successful, then the data-path of the TCP socket
is implemented by the kernel.
We choose to set this option at the earliest - just after CCS is complete.
Signed-off-by: Boris Pismenny <borisp@mellanox.com>
Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Paul Yang <yang.yang@baishancloud.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5253)
BIO_ctrl, BIO_callback_ctrl, BIO_ptr_ctrl, BIO_int_ctrl, BIO_reset,
BIO_seek, BIO_tell, BIO_flush, BIO_eof, BIO_set_close, BIO_get_close,
BIO_pending, BIO_wpending, BIO_ctrl_pending, BIO_ctrl_wpending,
-BIO_get_info_callback, BIO_set_info_callback, BIO_info_cb
+BIO_get_info_callback, BIO_set_info_callback, BIO_info_cb, BIO_get_ktls_send
- BIO control operations
=head1 SYNOPSIS
int BIO_get_info_callback(BIO *b, BIO_info_cb **cbp);
int BIO_set_info_callback(BIO *b, BIO_info_cb *cb);
+ int BIO_get_ktls_send(BIO *b);
+
=head1 DESCRIPTION
BIO_ctrl(), BIO_callback_ctrl(), BIO_ptr_ctrl() and BIO_int_ctrl()
return a size_t type and are functions, BIO_pending() and BIO_wpending() are
macros which call BIO_ctrl().
+BIO_get_ktls_send() return 1 if the BIO is using the Kernel TLS data-path for
+sending. Otherwise, it returns zero.
+
=head1 RETURN VALUES
BIO_reset() normally returns 1 for success and 0 or -1 for failure. File
BIO_pending(), BIO_ctrl_pending(), BIO_wpending() and BIO_ctrl_wpending()
return the amount of pending data.
+BIO_get_ktls_send() return 1 if the BIO is using the Kernel TLS data-path for
+sending. Otherwise, it returns zero.
+
=head1 NOTES
BIO_flush(), because it can write data may return 0 or -1 indicating
supported, if an error occurred, if EOF has not been reached and in
the case of BIO_seek() on a file BIO for a successful operation.
+=head1 HISTORY
+
+The BIO_get_ktls_send() function was added in OpenSSL 3.0.0.
+
=head1 COPYRIGHT
Copyright 2000-2016 The OpenSSL Project Authors. All Rights Reserved.
SSL_ERROR_WANT_ASYNC with this mode set if an asynchronous capable engine is
used to perform cryptographic operations. See L<SSL_get_error(3)>.
+=item SSL_MODE_NO_KTLS_TX
+
+Disable the use of the kernel TLS egress data-path.
+By default kernel TLS is enabled if it is supported by the negotiated ciphersuites
+and extensions and OpenSSL has been compiled with support for it.
+The kernel TLS data-path implements the record layer,
+and the crypto algorithm. The kernel will utilize the best hardware
+available for crypto. Using the kernel data-path should reduce the memory
+footprint of OpenSSL because no buffering is required. Also, the throughput
+should improve because data copy is avoided when user data is encrypted into
+kernel memory instead of the usual encrypt than copy to kernel.
+
+Kernel TLS might not support all the features of OpenSSL. For instance,
+renegotiation, and setting the maximum fragment size is not possible as of
+Linux 4.20.
+
=back
All modes are off by default except for SSL_MODE_AUTO_RETRY which is on by
=head1 HISTORY
SSL_MODE_ASYNC was first added to OpenSSL 1.1.0.
+SSL_MODE_NO_KTLS_TX was first added to OpenSSL 3.0.0.
=head1 COPYRIGHT
* Support Asynchronous operation
*/
# define SSL_MODE_ASYNC 0x00000100U
+/*
+ * Use the kernel TLS transmission data-path.
+ */
+# define SSL_MODE_NO_KTLS_TX 0x00000200U
/* Cert related flags */
/*
s->s3->empty_fragment_done = 1;
}
+ if (BIO_get_ktls_send(s->wbio)) {
+ /*
+ * ktls doesn't modify the buffer, but to avoid a warning we need to
+ * discard the const qualifier.
+ * This doesn't leak memory because the buffers have been released when
+ * switching to ktls.
+ */
+ SSL3_BUFFER_set_buf(&s->rlayer.wbuf[0], (unsigned char *)buf);
+ SSL3_BUFFER_set_offset(&s->rlayer.wbuf[0], 0);
+ goto wpacket_init_complete;
+ }
+
if (create_empty_fragment) {
wb = &s->rlayer.wbuf[0];
#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0
}
}
+ wpacket_init_complete:
+
totlen = 0;
/* Clear our SSL3_RECORD structures */
memset(wr, 0, sizeof(wr));
if (s->compress != NULL)
maxcomplen += SSL3_RT_MAX_COMPRESSED_OVERHEAD;
- /* write the header */
- if (!WPACKET_put_bytes_u8(thispkt, rectype)
+ /*
+ * When using offload kernel will write the header.
+ * Otherwise write the header now
+ */
+ if (!BIO_get_ktls_send(s->wbio)
+ && (!WPACKET_put_bytes_u8(thispkt, rectype)
|| !WPACKET_put_bytes_u16(thispkt, version)
|| !WPACKET_start_sub_packet_u16(thispkt)
|| (eivlen > 0
&& !WPACKET_allocate_bytes(thispkt, eivlen, NULL))
|| (maxcomplen > 0
&& !WPACKET_reserve_bytes(thispkt, maxcomplen,
- &compressdata))) {
+ &compressdata)))) {
SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
ERR_R_INTERNAL_ERROR);
goto err;
goto err;
}
} else {
- if (!WPACKET_memcpy(thispkt, thiswr->input, thiswr->length)) {
- SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
- ERR_R_INTERNAL_ERROR);
- goto err;
+ if (BIO_get_ktls_send(s->wbio)) {
+ SSL3_RECORD_reset_data(&wr[j]);
+ } else {
+ if (!WPACKET_memcpy(thispkt, thiswr->input, thiswr->length)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
+ ERR_R_INTERNAL_ERROR);
+ goto err;
+ }
+ SSL3_RECORD_reset_input(&wr[j]);
}
- SSL3_RECORD_reset_input(&wr[j]);
}
if (SSL_TREAT_AS_TLS13(s)
* This will be at most one cipher block or the tag length if using
* AEAD. SSL_RT_MAX_CIPHER_BLOCK_SIZE covers either case.
*/
- if (!WPACKET_reserve_bytes(thispkt, SSL_RT_MAX_CIPHER_BLOCK_SIZE,
- NULL)
- /*
- * We also need next the amount of bytes written to this
- * sub-packet
- */
+ if (!BIO_get_ktls_send(s->wbio)) {
+ if (!WPACKET_reserve_bytes(thispkt,
+ SSL_RT_MAX_CIPHER_BLOCK_SIZE,
+ NULL)
+ /*
+ * We also need next the amount of bytes written to this
+ * sub-packet
+ */
|| !WPACKET_get_length(thispkt, &len)) {
SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
ERR_R_INTERNAL_ERROR);
goto err;
- }
-
- /* Get a pointer to the start of this record excluding header */
- recordstart = WPACKET_get_curr(thispkt) - len;
+ }
- SSL3_RECORD_set_data(thiswr, recordstart);
- SSL3_RECORD_reset_input(thiswr);
- SSL3_RECORD_set_length(thiswr, len);
+ /* Get a pointer to the start of this record excluding header */
+ recordstart = WPACKET_get_curr(thispkt) - len;
+ SSL3_RECORD_set_data(thiswr, recordstart);
+ SSL3_RECORD_reset_input(thiswr);
+ SSL3_RECORD_set_length(thiswr, len);
+ }
}
if (s->statem.enc_write_state == ENC_WRITE_STATE_WRITE_PLAIN_ALERTS) {
goto err;
}
} else {
- if (s->method->ssl3_enc->enc(s, wr, numpipes, 1) < 1) {
- if (!ossl_statem_in_error(s)) {
- SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
- ERR_R_INTERNAL_ERROR);
+ if (!BIO_get_ktls_send(s->wbio)) {
+ if (s->method->ssl3_enc->enc(s, wr, numpipes, 1) < 1) {
+ if (!ossl_statem_in_error(s)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
+ ERR_R_INTERNAL_ERROR);
+ }
+ goto err;
}
- goto err;
}
}
thispkt = &pkt[j];
thiswr = &wr[j];
+ if (BIO_get_ktls_send(s->wbio))
+ goto mac_done;
+
/* Allocate bytes for the encryption overhead */
if (!WPACKET_get_length(thispkt, &origlen)
/* Encryption should never shrink the data! */
|| origlen > thiswr->length
|| (thiswr->length > origlen
&& !WPACKET_allocate_bytes(thispkt,
- thiswr->length - origlen, NULL))) {
+ thiswr->length - origlen,
+ NULL))) {
SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_DO_SSL3_WRITE,
ERR_R_INTERNAL_ERROR);
goto err;
goto err;
}
- /*
- * we should now have thiswr->data pointing to the encrypted data, which
- * is thiswr->length long
- */
- SSL3_RECORD_set_type(thiswr, type); /* not needed but helps for
- * debugging */
- SSL3_RECORD_add_length(thiswr, SSL3_RT_HEADER_LENGTH);
+ /* header is added by the kernel when using offload */
+ SSL3_RECORD_add_length(&wr[j], SSL3_RT_HEADER_LENGTH);
if (create_empty_fragment) {
/*
return 1;
}
+ mac_done:
+ /*
+ * we should now have thiswr->data pointing to the encrypted data, which
+ * is thiswr->length long
+ */
+ SSL3_RECORD_set_type(thiswr, type); /* not needed but helps for
+ * debugging */
+
/* now let's set up wb */
SSL3_BUFFER_set_left(&s->rlayer.wbuf[j],
prefix_len + SSL3_RECORD_get_length(thiswr));
clear_sys_error();
if (s->wbio != NULL) {
s->rwstate = SSL_WRITING;
+
+ /*
+ * To prevent coalescing of control and data messages,
+ * such as in buffer_write, we flush the BIO
+ */
+ if (BIO_get_ktls_send(s->wbio) && type != SSL3_RT_APPLICATION_DATA) {
+ i = BIO_flush(s->wbio);
+ if (i <= 0)
+ return i;
+ }
+
+ if (BIO_get_ktls_send(s->wbio)
+ && type != SSL3_RT_APPLICATION_DATA) {
+ BIO_set_ktls_ctrl_msg(s->wbio, type);
+ }
/* TODO(size_t): Convert this call */
i = BIO_write(s->wbio, (char *)
&(SSL3_BUFFER_get_buf(&wb[currbuf])
#define SSL3_RECORD_get_input(r) ((r)->input)
#define SSL3_RECORD_set_input(r, i) ((r)->input = (i))
#define SSL3_RECORD_reset_input(r) ((r)->input = (r)->data)
+#define SSL3_RECORD_reset_data(r) ((r)->data = (r)->input)
#define SSL3_RECORD_get_seq_num(r) ((r)->seq_num)
#define SSL3_RECORD_get_off(r) ((r)->off)
#define SSL3_RECORD_set_off(r, o) ((r)->off = (o))
for (currpipe = 0; currpipe < numwpipes; currpipe++) {
SSL3_BUFFER *thiswb = &wb[currpipe];
- if (thiswb->buf != NULL && thiswb->len != len) {
+ if (thiswb->len != len) {
OPENSSL_free(thiswb->buf);
thiswb->buf = NULL; /* force reallocation */
}
if (thiswb->buf == NULL) {
- p = OPENSSL_malloc(len);
- if (p == NULL) {
- s->rlayer.numwpipes = currpipe;
- /*
- * We've got a malloc failure, and we're still initialising
- * buffers. We assume we're so doomed that we won't even be able
- * to send an alert.
- */
- SSLfatal(s, SSL_AD_NO_ALERT,
- SSL_F_SSL3_SETUP_WRITE_BUFFER, ERR_R_MALLOC_FAILURE);
- return 0;
+ if (s->wbio == NULL || !BIO_get_ktls_send(s->wbio)) {
+ p = OPENSSL_malloc(len);
+ if (p == NULL) {
+ s->rlayer.numwpipes = currpipe;
+ /*
+ * We've got a malloc failure, and we're still initialising
+ * buffers. We assume we're so doomed that we won't even be able
+ * to send an alert.
+ */
+ SSLfatal(s, SSL_AD_NO_ALERT,
+ SSL_F_SSL3_SETUP_WRITE_BUFFER, ERR_R_MALLOC_FAILURE);
+ return 0;
+ }
+ } else {
+ p = NULL;
}
memset(thiswb, 0, sizeof(SSL3_BUFFER));
thiswb->buf = p;
while (pipes > 0) {
wb = &RECORD_LAYER_get_wbuf(&s->rlayer)[pipes - 1];
- OPENSSL_free(wb->buf);
+ if (s->wbio == NULL || !BIO_get_ktls_send(s->wbio))
+ OPENSSL_free(wb->buf);
wb->buf = NULL;
pipes--;
}
#include <openssl/ct.h>
#include "internal/cryptlib.h"
#include "internal/refcount.h"
+#include "internal/ktls.h"
static int ssl_undefined_function_1(SSL *ssl, SSL3_RECORD *r, size_t s, int t)
{
dane_final(&s->dane);
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_SSL, s, &s->ex_data);
+ RECORD_LAYER_release(&s->rlayer);
+
/* Ignore return value */
ssl_free_wbio_buffer(s);
BIO_free_all(s->wbio);
+ s->wbio = NULL;
BIO_free_all(s->rbio);
+ s->rbio = NULL;
BUF_MEM_free(s->init_buf);
if (s->method != NULL)
s->method->ssl_free(s);
- RECORD_LAYER_release(&s->rlayer);
-
SSL_CTX_free(s->ctx);
ASYNC_WAIT_CTX_free(s->waitctx);
}
BIO_set_fd(bio, fd, BIO_NOCLOSE);
SSL_set_bio(s, bio, bio);
+#ifndef OPENSSL_NO_KTLS
+ /*
+ * The new socket is created successfully regardless of ktls_enable.
+ * ktls_enable doesn't change any functionality of the socket, except
+ * changing the setsockopt to enable the processing of ktls_start.
+ * Thus, it is not a problem to call it for non-TLS sockets.
+ */
+ ktls_enable(fd);
+#endif /* OPENSSL_NO_KTLS */
ret = 1;
err:
return ret;
}
BIO_set_fd(bio, fd, BIO_NOCLOSE);
SSL_set0_wbio(s, bio);
+#ifndef OPENSSL_NO_KTLS
+ /*
+ * The new socket is created successfully regardless of ktls_enable.
+ * ktls_enable doesn't change any functionality of the socket, except
+ * changing the setsockopt to enable the processing of ktls_start.
+ * Thus, it is not a problem to call it for non-TLS sockets.
+ */
+ ktls_enable(fd);
+#endif /* OPENSSL_NO_KTLS */
} else {
BIO_up_ref(rbio);
SSL_set0_wbio(s, rbio);
case SSL_CTRL_SET_MAX_SEND_FRAGMENT:
if (larg < 512 || larg > SSL3_RT_MAX_PLAIN_LENGTH)
return 0;
+#ifndef OPENSSL_NO_KTLS
+ if (s->wbio != NULL && BIO_get_ktls_send(s->wbio))
+ return 0;
+#endif /* OPENSSL_NO_KTLS */
s->max_send_fragment = larg;
if (s->max_send_fragment < s->split_send_fragment)
s->split_send_fragment = s->max_send_fragment;
# include "internal/dane.h"
# include "internal/refcount.h"
# include "internal/tsan_assist.h"
+# include "internal/bio.h"
# ifdef OPENSSL_BUILD_SHLIBSSL
# undef OPENSSL_EXTERN
#include <stdio.h>
#include "ssl_locl.h"
+#include "record/record_locl.h"
+#include "internal/ktls.h"
+#include "internal/cryptlib.h"
#include <openssl/comp.h>
#include <openssl/evp.h>
#include <openssl/kdf.h>
#include <openssl/rand.h>
+#include <openssl/obj_mac.h>
/* seed1 through seed5 are concatenated */
static int tls1_PRF(SSL *s,
EVP_PKEY *mac_key;
size_t n, i, j, k, cl;
int reuse_dd = 0;
+#ifndef OPENSSL_NO_KTLS
+ struct tls12_crypto_info_aes_gcm_128 crypto_info;
+ BIO *wbio;
+ unsigned char geniv[12];
+#endif
c = s->s3->tmp.new_sym_enc;
m = s->s3->tmp.new_hash;
ERR_R_INTERNAL_ERROR);
goto err;
}
+#ifndef OPENSSL_NO_KTLS
+ if (s->compress)
+ goto skip_ktls;
+
+ if ((which & SSL3_CC_READ) ||
+ ((which & SSL3_CC_WRITE) && (s->mode & SSL_MODE_NO_KTLS_TX)))
+ goto skip_ktls;
+
+ /* ktls supports only the maximum fragment size */
+ if (ssl_get_max_send_fragment(s) != SSL3_RT_MAX_PLAIN_LENGTH)
+ goto skip_ktls;
+
+ /* check that cipher is AES_GCM_128 */
+ if (EVP_CIPHER_nid(c) != NID_aes_128_gcm
+ || EVP_CIPHER_mode(c) != EVP_CIPH_GCM_MODE
+ || EVP_CIPHER_key_length(c) != TLS_CIPHER_AES_GCM_128_KEY_SIZE)
+ goto skip_ktls;
+
+ /* check version is 1.2 */
+ if (s->version != TLS1_2_VERSION)
+ goto skip_ktls;
+
+ wbio = s->wbio;
+ if (!ossl_assert(wbio != NULL)) {
+ SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS1_CHANGE_CIPHER_STATE,
+ ERR_R_INTERNAL_ERROR);
+ goto err;
+ }
+
+ /* All future data will get encrypted by ktls. Flush the BIO or skip ktls */
+ if (BIO_flush(wbio) <= 0)
+ goto skip_ktls;
+
+ /* ktls doesn't support renegotiation */
+ if (BIO_get_ktls_send(s->wbio)) {
+ SSLfatal(s, SSL_AD_NO_RENEGOTIATION, SSL_F_TLS1_CHANGE_CIPHER_STATE,
+ ERR_R_INTERNAL_ERROR);
+ goto err;
+ }
+
+ memset(&crypto_info, 0, sizeof(crypto_info));
+ crypto_info.info.cipher_type = TLS_CIPHER_AES_GCM_128;
+ crypto_info.info.version = s->version;
+
+ EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_GET_IV,
+ EVP_GCM_TLS_FIXED_IV_LEN + EVP_GCM_TLS_EXPLICIT_IV_LEN,
+ geniv);
+ memcpy(crypto_info.iv, geniv + EVP_GCM_TLS_FIXED_IV_LEN,
+ TLS_CIPHER_AES_GCM_128_IV_SIZE);
+ memcpy(crypto_info.salt, geniv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
+ memcpy(crypto_info.key, key, EVP_CIPHER_key_length(c));
+ memcpy(crypto_info.rec_seq, &s->rlayer.write_sequence,
+ TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
+
+ /* ktls works with user provided buffers directly */
+ if (BIO_set_ktls(wbio, &crypto_info, which & SSL3_CC_WRITE)) {
+ ssl3_release_write_buffer(s);
+ SSL_set_options(s, SSL_OP_NO_RENEGOTIATION);
+ }
+
+ skip_ktls:
+#endif /* OPENSSL_NO_KTLS */
s->statem.enc_write_state = ENC_WRITE_STATE_VALID;
#ifdef SSL_DEBUG
BIO_get_cipher_ctx define
BIO_get_cipher_status define
BIO_get_close define
+BIO_get_ktls_send define
BIO_get_conn_address define
BIO_get_conn_hostname define
BIO_get_conn_port define
projects / oweals / openssl.git / commitdiff