amd64: flush L1 data cache on syscall return with an error.

The knob allows to select the flushing mode or turn it off/on.  The
idea, as well as the list of the ignored syscall errors, were taken
from https://www.openwall.com/lists/kernel-hardening/2018/10/11/10 .

I was not able to measure statistically significant difference between
flush enabled vs disabled using syscall_timing getuid.

Reviewed by:	bwidawsk
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D17536

sys/amd64/amd64/initcpu.c

@@ -253,6 +253,7 @@ initializecpu(void)
 	}
 	hw_ibrs_recalculate();
 	hw_ssb_recalculate(false);
+	amd64_syscall_ret_flush_l1d_recalc();
 	switch (cpu_vendor_id) {
 	case CPU_VENDOR_AMD:
 		init_amd();

sys/amd64/amd64/machdep.c

@@ -1722,6 +1722,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	    != NULL)
 		vty_set_preferred(VTY_VT);
 
+	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
+	TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+	TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush",
+	    &syscall_ret_l1d_flush_mode);
+
 	finishidentcpu();	/* Final stage of CPU initialization */
 	initializecpu();	/* Initialize CPU registers */
 
@@ -1865,9 +1870,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 #endif
 	thread0.td_critnest = 0;
 
-	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
-	TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
-
 	TSEXIT();
 
 	/* Location of kernel stack for locore */

sys/amd64/amd64/support.S

@@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw)
 	ret
 #undef	L1D_FLUSH_SIZE
 END(flush_l1d_sw)
+
+ENTRY(flush_l1d_sw_abi)
+	pushq	%rbx
+	call	flush_l1d_sw
+	popq	%rbx
+	ret
+END(flush_l1d_sw_abi)

sys/amd64/amd64/trap.c

@@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td)
 
 #include "../../kern/subr_syscall.c"
 
+static void (*syscall_ret_l1d_flush)(void);
+int syscall_ret_l1d_flush_mode;
+
+static void
+flush_l1d_hw(void)
+{
+
+	wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
+}
+
+static void __inline
+amd64_syscall_ret_flush_l1d_inline(int error)
+{
+	void (*p)(void);
+
+	if (error != 0 && error != EEXIST && error != EAGAIN &&
+	    error != EXDEV && error != ENOENT && error != ENOTCONN &&
+	    error != EINPROGRESS) {
+		p = syscall_ret_l1d_flush;
+		if (p != NULL)
+			p();
+	}
+}
+
+void
+amd64_syscall_ret_flush_l1d(int error)
+{
+
+	amd64_syscall_ret_flush_l1d_inline(error);
+}
+
+void
+amd64_syscall_ret_flush_l1d_recalc(void)
+{
+	bool l1d_hw;
+
+	l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
+again:
+	switch (syscall_ret_l1d_flush_mode) {
+	case 0:
+		syscall_ret_l1d_flush = NULL;
+		break;
+	case 1:
+		syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
+		    flush_l1d_sw_abi;
+		break;
+	case 2:
+		syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
+		break;
+	case 3:
+		syscall_ret_l1d_flush = flush_l1d_sw_abi;
+		break;
+	default:
+		syscall_ret_l1d_flush_mode = 1;
+		goto again;
+	}
+}
+
+static int
+machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
+{
+	int error, val;
+
+	val = syscall_ret_l1d_flush_mode;
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	syscall_ret_l1d_flush_mode = val;
+	amd64_syscall_ret_flush_l1d_recalc();
+	return (0);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
+    CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
+    machdep_syscall_ret_flush_l1d, "I",
+    "Flush L1D on syscall return with error (0 - off, 1 - on, "
+    "2 - use hw only, 3 - use sw only");
+
+
 /*
  * System call handler for native binaries.  The trap frame is already
  * set up by the assembler trampoline and a pointer to it is saved in
@@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced)
 	 */
 	if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
 		set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+
+	amd64_syscall_ret_flush_l1d_inline(error);
 }

sys/amd64/ia32/ia32_syscall.c

@@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame)
 	}
 
 	syscallret(td, error);
+	amd64_syscall_ret_flush_l1d(error);
 }
 
 static void

sys/amd64/include/md_var.h

@@ -41,6 +41,7 @@ extern int	hw_lower_amd64_sharedpage;
 extern int	hw_ibrs_disable;
 extern int	hw_ssb_disable;
 extern int	nmi_flush_l1d_sw;
+extern int	syscall_ret_l1d_flush_mode;
 
 /*
  * The file "conf/ldscript.amd64" defines the symbol "kernphys".  Its
@@ -55,8 +56,11 @@ void	amd64_conf_fast_syscall(void);
 void	amd64_db_resume_dbreg(void);
 void	amd64_lower_shared_page(struct sysentvec *);
 void	amd64_syscall(struct thread *td, int traced);
+void	amd64_syscall_ret_flush_l1d(int error);
+void	amd64_syscall_ret_flush_l1d_recalc(void);
 void	doreti_iret(void) __asm(__STRING(doreti_iret));
 void	doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void	flush_l1d_sw_abi(void);
 void	ld_ds(void) __asm(__STRING(ld_ds));
 void	ld_es(void) __asm(__STRING(ld_es));
 void	ld_fs(void) __asm(__STRING(ld_fs));

sys/dev/cpuctl/cpuctl.c

@@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td)
 	hw_ibrs_recalculate();
 	restore_cpu(oldcpu, is_bound, td);
 	hw_ssb_recalculate(true);
+#ifdef __amd64__
+	amd64_syscall_ret_flush_l1d_recalc();
+#endif
 	printcpuinfo();
 	return (0);
 }