Add support for the fpu_kern(9) KPI on arm64. It hooks into the existing

VFP code to store the old context, with lazy loading of the new context
when needed.

FPU_KERN_NOCTX is missing as this is unused in the crypto code this has
been tested with, and I am unsure on the requirements of the UEFI
Runtime Services.

Reviewed by:	kib
Obtained from:	ABT Systeems Ltd
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D8276

share/man/man9/fpu_kern.9

@@ -23,7 +23,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 23, 2014
+.Dd October 20, 2016
 .Dt FPU_KERN 9
 .Os
 .Sh NAME
@@ -134,11 +134,11 @@ of increased system latency.
 .El
 .Pp
 The function does not sleep or block.
-It could cause the
+It could cause an FPU trap during execution, and on the first FPU access
+after the function returns, as well as after each context switch.
+On i386 and amd64 this will be the
 .Nm Device Not Available
-exception during execution, and on the first FPU access after the
-function returns, as well as after each context switch
-(see Intel Software Developer Manual for the reference).
+exception (see Intel Software Developer Manual for the reference).
 Currently, no errors are defined which can be returned by
 .Fn fpu_kern_enter
 to the caller.
@@ -190,7 +190,7 @@ and false otherwise.
 .Sh NOTES
 The
 .Nm
-is currently implemented only for the i386 and amd64 architectures.
+is currently implemented only for the i386, amd64, and arm64 architectures.
 .Pp
 There is no way to handle floating point exceptions raised from
 kernel mode.
@@ -208,3 +208,5 @@ The
 .Nm
 facitily and this manual page were written by
 .An Konstantin Belousov Aq Mt kib@FreeBSD.org .
+The arm64 support was added by
+.An Andrew Turner Aq Mt andrew@FreeBSD.org .

sys/arm64/arm64/trap.c

@@ -282,9 +282,17 @@ do_el1h_sync(struct trapframe *frame)
 	switch(exception) {
 	case EXCP_FP_SIMD:
 	case EXCP_TRAP_FP:
-		print_registers(frame);
-		printf(" esr:         %.8lx\n", esr);
-		panic("VFP exception in the kernel");
+#ifdef VFP
+		if ((curthread->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) {
+			vfp_restore_state();
+		} else
+#endif
+		{
+			print_registers(frame);
+			printf(" esr:         %.8lx\n", esr);
+			panic("VFP exception in the kernel");
+		}
+		break;
 	case EXCP_INSN_ABORT:
 	case EXCP_DATA_ABORT:
 		far = READ_SPECIALREG(far_el1);
@@ -412,6 +420,9 @@ do_el0_sync(struct trapframe *frame)
 
 	KASSERT((curthread->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
 	    ("Kernel VFP flags set while entering userspace"));
+	KASSERT(
+	    curthread->td_pcb->pcb_fpusaved == &curthread->td_pcb->pcb_fpustate,
+	    ("Kernel VFP state in use when entering userspace"));
 }
 
 void

sys/arm64/arm64/vfp.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015 The FreeBSD Foundation
+ * Copyright (c) 2015-2016 The FreeBSD Foundation
  * All rights reserved.
  *
  * This software was developed by Andrew Turner under
@@ -48,6 +48,14 @@ CTASSERT(sizeof(((struct pcb *)0)->pcb_fpustate.vfp_regs) == 16 * 32);
 static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx",
     "Kernel contexts for VFP state");
 
+struct fpu_kern_ctx {
+	struct vfpstate	*prev;
+#define	FPU_KERN_CTX_DUMMY	0x01	/* avoided save for the kern thread */
+#define	FPU_KERN_CTX_INUSE	0x02
+	uint32_t	 flags;
+	struct vfpstate	 state;
+};
+
 static void
 vfp_enable(void)
 {
@@ -71,9 +79,10 @@ vfp_disable(void)
 }
 
 /*
- * Called when the thread is dying. If the thread was the last to use the
- * VFP unit mark it as unused to tell the kernel the fp state is unowned.
- * Ensure the VFP unit is off so we get an exception on the next access.
+ * Called when the thread is dying or when discarding the kernel VFP state.
+ * If the thread was the last to use the VFP unit mark it as unused to tell
+ * the kernel the fp state is unowned. Ensure the VFP unit is off so we get
+ * an exception on the next access.
  */
 void
 vfp_discard(struct thread *td)
@@ -226,4 +235,111 @@ vfp_init(void)
 
 SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL);
 
+struct fpu_kern_ctx *
+fpu_kern_alloc_ctx(u_int flags)
+{
+	struct fpu_kern_ctx *res;
+	size_t sz;
+
+	sz = sizeof(struct fpu_kern_ctx);
+	res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ?
+	    M_NOWAIT : M_WAITOK) | M_ZERO);
+	return (res);
+}
+
+void
+fpu_kern_free_ctx(struct fpu_kern_ctx *ctx)
+{
+
+	KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx"));
+	/* XXXAndrew clear the memory ? */
+	free(ctx, M_FPUKERN_CTX);
+}
+
+int
+fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
+{
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+	KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
+	    ("using inuse ctx"));
+
+	if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) {
+		ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE;
+		return (0);
+	}
+	/*
+	 * Check either we are already using the VFP in the kernel, or
+	 * the the saved state points to the default user space.
+	 */
+	KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0 ||
+	    pcb->pcb_fpusaved == &pcb->pcb_fpustate,
+	    ("Mangled pcb_fpusaved %x %p %p", pcb->pcb_fpflags, pcb->pcb_fpusaved, &pcb->pcb_fpustate));
+	ctx->flags = FPU_KERN_CTX_INUSE;
+	vfp_save_state(curthread, pcb);
+	ctx->prev = pcb->pcb_fpusaved;
+	pcb->pcb_fpusaved = &ctx->state;
+	pcb->pcb_fpflags |= PCB_FP_KERN;
+	pcb->pcb_fpflags &= ~PCB_FP_STARTED;
+
+	return (0);
+}
+
+int
+fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
+{
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+
+	KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0,
+	    ("FPU context not inuse"));
+	ctx->flags &= ~FPU_KERN_CTX_INUSE;
+
+	if (is_fpu_kern_thread(0) &&
+	    (ctx->flags & FPU_KERN_CTX_DUMMY) != 0)
+		return (0);
+	KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx"));
+	critical_enter();
+	vfp_discard(td);
+	critical_exit();
+	pcb->pcb_fpflags &= ~PCB_FP_STARTED;
+	pcb->pcb_fpusaved = ctx->prev;
+
+	if (pcb->pcb_fpusaved == &pcb->pcb_fpustate) {
+		pcb->pcb_fpflags &= ~PCB_FP_KERN;
+	} else {
+		KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0,
+		    ("unpaired fpu_kern_leave"));
+	}
+
+	return (0);
+}
+
+int
+fpu_kern_thread(u_int flags)
+{
+	struct pcb *pcb = curthread->td_pcb;
+
+	KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
+	    ("Only kthread may use fpu_kern_thread"));
+	KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
+	    ("Mangled pcb_fpusaved"));
+	KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) == 0,
+	    ("Thread already setup for the VFP"));
+	pcb->pcb_fpflags |= PCB_FP_KERN;
+	return (0);
+}
+
+int
+is_fpu_kern_thread(u_int flags)
+{
+	struct pcb *curpcb;
+
+	if ((curthread->td_pflags & TDP_KTHREAD) == 0)
+		return (0);
+	curpcb = curthread->td_pcb;
+	return ((curpcb->pcb_fpflags & PCB_FP_KERN) != 0);
+}
 #endif

sys/arm64/include/pcb.h

@@ -54,6 +54,7 @@ struct pcb {
 	struct vfpstate	*pcb_fpusaved;
 	int		pcb_fpflags;
 #define	PCB_FP_STARTED	0x01
+#define	PCB_FP_KERN	0x02
 /* The bits passed to userspace in get_fpcontext */
 #define	PCB_FP_USERMASK	(PCB_FP_STARTED)
 	u_int		pcb_vfpcpu;	/* Last cpu this thread ran VFP code */

sys/arm64/include/vfp.h

@@ -45,6 +45,23 @@ void	vfp_init(void);
 void	vfp_discard(struct thread *);
 void	vfp_restore_state(void);
 void	vfp_save_state(struct thread *, struct pcb *);
+
+struct fpu_kern_ctx;
+
+/*
+ * Flags for fpu_kern_alloc_ctx(), fpu_kern_enter() and fpu_kern_thread().
+ */
+#define	FPU_KERN_NORMAL	0x0000
+#define	FPU_KERN_NOWAIT	0x0001
+#define	FPU_KERN_KTHR	0x0002
+
+struct fpu_kern_ctx *fpu_kern_alloc_ctx(u_int);
+void fpu_kern_free_ctx(struct fpu_kern_ctx *);
+int fpu_kern_enter(struct thread *, struct fpu_kern_ctx *, u_int);
+int fpu_kern_leave(struct thread *, struct fpu_kern_ctx *);
+int fpu_kern_thread(u_int);
+int is_fpu_kern_thread(u_int);
+
 #endif
 
 #endif

sys/opencrypto/crypto.c

@@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include "cryptodev_if.h"
 
-#if defined(__i386__) || defined(__amd64__)
+#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
 #include <machine/pcb.h>
 #endif
 
@@ -1246,7 +1246,7 @@ crypto_proc(void)
 	u_int32_t hid;
 	int result, hint;
 
-#if defined(__i386__) || defined(__amd64__)
+#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
 	fpu_kern_thread(FPU_KERN_NORMAL);
 #endif