mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-12-12 09:58:36 +00:00
Code Issues Releases Activity

Add a TOE KTLS mode and a TOE hook for allocating TLS sessions.

Browse Source 
This adds the glue to allocate TLS sessions and invokes it from
the TLS enable socket option handler.  This also adds some counters
for active TOE sessions.

The TOE KTLS mode is returned by getsockopt(TLSTX_TLS_MODE) when
TOE KTLS is in use on a socket, but cannot be set via setsockopt().

To simplify various checks, a TLS session now includes an explicit
'mode' member set to the value returned by TLSTX_TLS_MODE.  Various
places that used to check 'sw_encrypt' against NULL to determine
software vs ifnet (NIC) TLS now check 'mode' instead.

Reviewed by:	np, gallatin
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D21891
This commit is contained in:
John Baldwin 2019-10-08 21:34:06 +00:00
parent fa43c5d49e
commit 9e14430d46
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=353328
10 changed files with 140 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
sys/kern/kern_sendfile.c
View File

@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
#include <sys/vnode.h>
#include <net/vnet.h>
#include <netinet/tcp.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
@ -295,7 +296,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, int error)
mb_free_notready(sfio->m, sfio->npages);
#ifdef KERN_TLS
} else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) {
} else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
/*
* I/O operation is complete, but we still need to
* encrypt. We cannot do this in the interrupt thread
@ -1028,7 +1029,7 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
*/
free(sfio, M_TEMP);
#ifdef KERN_TLS
if (tls != NULL && tls->sw_encrypt != NULL) {
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);
soref(so);

121
sys/kern/uipc_ktls.c
View File

@ -63,6 +63,9 @@ __FBSDID("$FreeBSD$");
#include <netinet/in_pcb.h>
#endif
#include <netinet/tcp_var.h>
#ifdef TCP_OFFLOAD
#include <netinet/tcp_offload.h>
#endif
#include <opencrypto/xform.h>
#include <vm/uma_dbg.h>
#include <vm/vm.h>
@ -161,6 +164,10 @@ SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD, 0,
"Software TLS session stats");
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0,
"Hardware (ifnet) TLS session stats");
#ifdef TCP_OFFLOAD
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0,
"TOE TLS session stats");
#endif
static counter_u64_t ktls_sw_cbc;
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc,
@ -199,6 +206,18 @@ SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN,
&ktls_ifnet_permitted, 1,
"Whether to permit hardware (ifnet) TLS sessions");
#ifdef TCP_OFFLOAD
static counter_u64_t ktls_toe_cbc;
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD,
&ktls_toe_cbc,
"Active number of TOE TLS sessions using AES-CBC");
static counter_u64_t ktls_toe_gcm;
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD,
&ktls_toe_gcm,
"Active number of TOE TLS sessions using AES-GCM");
#endif
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS");
static void ktls_cleanup(struct ktls_session *tls);
@ -325,6 +344,10 @@ ktls_init(void *dummy __unused)
ktls_ifnet_reset = counter_u64_alloc(M_WAITOK);
ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK);
ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK);
#ifdef TCP_OFFLOAD
ktls_toe_cbc = counter_u64_alloc(M_WAITOK);
ktls_toe_gcm = counter_u64_alloc(M_WAITOK);
#endif
rm_init(&ktls_backends_lock, "ktls backends");
LIST_INIT(&ktls_backends);
@ -607,7 +630,8 @@ ktls_cleanup(struct ktls_session *tls)
{
counter_u64_add(ktls_offload_active, -1);
if (tls->free != NULL) {
switch (tls->mode) {
case TCP_TLS_MODE_SW:
MPASS(tls->be != NULL);
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
@ -618,7 +642,8 @@ ktls_cleanup(struct ktls_session *tls)
break;
}
tls->free(tls);
} else if (tls->snd_tag != NULL) {
break;
case TCP_TLS_MODE_IFNET:
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_ifnet_cbc, -1);
@ -628,6 +653,19 @@ ktls_cleanup(struct ktls_session *tls)
break;
}
m_snd_tag_rele(tls->snd_tag);
break;
#ifdef TCP_OFFLOAD
case TCP_TLS_MODE_TOE:
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_toe_cbc, -1);
break;
case CRYPTO_AES_NIST_GCM_16:
counter_u64_add(ktls_toe_gcm, -1);
break;
}
break;
#endif
}
if (tls->params.auth_key != NULL) {
explicit_bzero(tls->params.auth_key, tls->params.auth_key_len);
@ -646,6 +684,52 @@ ktls_cleanup(struct ktls_session *tls)
}
#if defined(INET) || defined(INET6)
#ifdef TCP_OFFLOAD
static int
ktls_try_toe(struct socket *so, struct ktls_session *tls)
{
struct inpcb *inp;
struct tcpcb *tp;
int error;
inp = so->so_pcb;
INP_WLOCK(inp);
if (inp->inp_flags2 & INP_FREED) {
INP_WUNLOCK(inp);
return (ECONNRESET);
}
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
INP_WUNLOCK(inp);
return (ECONNRESET);
}
if (inp->inp_socket == NULL) {
INP_WUNLOCK(inp);
return (ECONNRESET);
}
tp = intotcpcb(inp);
if (tp->tod == NULL) {
INP_WUNLOCK(inp);
return (EOPNOTSUPP);
}
error = tcp_offload_alloc_tls_session(tp, tls);
INP_WUNLOCK(inp);
if (error == 0) {
tls->mode = TCP_TLS_MODE_TOE;
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_toe_cbc, 1);
break;
case CRYPTO_AES_NIST_GCM_16:
counter_u64_add(ktls_toe_gcm, 1);
break;
}
}
return (error);
}
#endif
/*
* Common code used when first enabling ifnet TLS on a connection or
* when allocating a new ifnet TLS session due to a routing change.
@ -744,6 +828,7 @@ ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force)
error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
if (error == 0) {
tls->mode = TCP_TLS_MODE_IFNET;
tls->snd_tag = mst;
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
@ -787,6 +872,7 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls)
rm_runlock(&ktls_backends_lock, &prio);
if (be == NULL)
return (EOPNOTSUPP);
tls->mode = TCP_TLS_MODE_SW;
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_sw_cbc, 1);
@ -834,8 +920,12 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
if (error)
return (error);
/* Prefer ifnet TLS over software TLS. */
error = ktls_try_ifnet(so, tls, false);
/* Prefer TOE -> ifnet TLS -> software TLS. */
#ifdef TCP_OFFLOAD
error = ktls_try_toe(so, tls);
if (error)
#endif
error = ktls_try_ifnet(so, tls, false);
if (error)
error = ktls_try_sw(so, tls);
@ -852,7 +942,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
SOCKBUF_LOCK(&so->so_snd);
so->so_snd.sb_tls_info = tls;
if (tls->sw_encrypt == NULL)
if (tls->mode != TCP_TLS_MODE_SW)
so->so_snd.sb_flags |= SB_TLS_IFNET;
SOCKBUF_UNLOCK(&so->so_snd);
sbunlock(&so->so_snd);
@ -875,10 +965,8 @@ ktls_get_tx_mode(struct socket *so)
tls = so->so_snd.sb_tls_info;
if (tls == NULL)
mode = TCP_TLS_MODE_NONE;
else if (tls->sw_encrypt != NULL)
mode = TCP_TLS_MODE_SW;
else
mode = TCP_TLS_MODE_IFNET;
mode = tls->mode;
SOCKBUF_UNLOCK(&so->so_snd);
return (mode);
}
@ -893,7 +981,13 @@ ktls_set_tx_mode(struct socket *so, int mode)
struct inpcb *inp;
int error;
MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET);
switch (mode) {
case TCP_TLS_MODE_SW:
case TCP_TLS_MODE_IFNET:
break;
default:
return (EINVAL);
}
inp = so->so_pcb;
INP_WLOCK_ASSERT(inp);
@ -904,8 +998,7 @@ ktls_set_tx_mode(struct socket *so, int mode)
return (0);
}
if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) ||
(tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) {
if (tls->mode == mode) {
SOCKBUF_UNLOCK(&so->so_snd);
return (0);
}
@ -952,7 +1045,7 @@ ktls_set_tx_mode(struct socket *so, int mode)
SOCKBUF_LOCK(&so->so_snd);
so->so_snd.sb_tls_info = tls_new;
if (tls_new->sw_encrypt == NULL)
if (tls_new->mode != TCP_TLS_MODE_SW)
so->so_snd.sb_flags |= SB_TLS_IFNET;
SOCKBUF_UNLOCK(&so->so_snd);
sbunlock(&so->so_snd);
@ -1238,7 +1331,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
* When using ifnet TLS, unencrypted TLS records are
* sent down the stack to the NIC.
*/
if (tls->sw_encrypt != NULL) {
if (tls->mode == TCP_TLS_MODE_SW) {
m->m_flags |= M_NOTREADY;
pgs->nrdy = pgs->npgs;
*enq_cnt += pgs->npgs;
@ -1278,7 +1371,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
pgs = m->m_ext.ext_pgs;
KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf"));
KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
pgs->enc_cnt = page_count;
pgs->mbuf = m;

4
sys/kern/uipc_socket.c
View File

@ -1491,7 +1491,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
tls_pruflag = 0;
tls = ktls_hold(so->so_snd.sb_tls_info);
if (tls != NULL) {
if (tls->sw_encrypt != NULL)
if (tls->mode == TCP_TLS_MODE_SW)
tls_pruflag = PRUS_NOTREADY;
if (control != NULL) {
@ -1659,7 +1659,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
}
#ifdef KERN_TLS
if (tls != NULL && tls->sw_encrypt != NULL) {
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
/*
* Note that error is intentionally
* ignored.

1
sys/netinet/tcp.h
View File

@ -357,6 +357,7 @@ struct tcp_function_set {
#define TCP_TLS_MODE_NONE 0
#define TCP_TLS_MODE_SW 1
#define TCP_TLS_MODE_IFNET 2
#define TCP_TLS_MODE_TOE 3
/*
* TCP Control message types

11
sys/netinet/tcp_offload.c
View File

@ -178,6 +178,17 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti)
tod->tod_tcp_info(tod, tp, ti);
}
int
tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
{
struct toedev *tod = tp->tod;
KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
INP_WLOCK_ASSERT(tp->t_inpcb);
return (tod->tod_alloc_tls_session(tod, tp, tls));
}
void
tcp_offload_detach(struct tcpcb *tp)
{

1
sys/netinet/tcp_offload.h
View File

@ -46,6 +46,7 @@ int tcp_offload_output(struct tcpcb *);
void tcp_offload_rcvd(struct tcpcb *);
void tcp_offload_ctloutput(struct tcpcb *, int, int);
void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
void tcp_offload_detach(struct tcpcb *);
#endif

2
sys/netinet/tcp_usrreq.c
View File

@ -1936,8 +1936,6 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
if (error)
return (error);
if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET)
return (EINVAL);
INP_WLOCK_RECHECK(inp);
error = ktls_set_tx_mode(so, ui);

9
sys/netinet/toecore.c
View File

@ -191,6 +191,14 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused,
return;
}
static int
toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
struct ktls_session *tls __unused)
{
return (EINVAL);
}
/*
* Inform one or more TOE devices about a listening socket.
*/
@ -281,6 +289,7 @@ init_toedev(struct toedev *tod)
tod->tod_offload_socket = toedev_offload_socket;
tod->tod_ctloutput = toedev_ctloutput;
tod->tod_tcp_info = toedev_tcp_info;
tod->tod_alloc_tls_session = toedev_alloc_tls_session;
}
/*

5
sys/netinet/toecore.h
View File

@ -41,6 +41,7 @@ struct tcpopt;
struct tcphdr;
struct in_conninfo;
struct tcp_info;
struct ktls_session;
struct toedev {
TAILQ_ENTRY(toedev) link; /* glue for toedev_list */
@ -108,6 +109,10 @@ struct toedev {
/* Update software state */
void (*tod_tcp_info)(struct toedev *, struct tcpcb *,
struct tcp_info *);
/* Create a TLS session */
int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
struct ktls_session *);
};
typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);

1
sys/sys/ktls.h
View File

@ -156,6 +156,7 @@ struct ktls_session {
struct tls_session_params params;
u_int wq_index;
volatile u_int refcount;
int mode;
struct task reset_tag_task;
struct inpcb *inp;