Add the KMSAN runtime

KMSAN enables the use of LLVM's MemorySanitizer in the kernel.  This
enables precise detection of uses of uninitialized memory.  As with
KASAN, this feature has substantial runtime overhead and is intended to
be used as part of some automated testing regime.

The runtime maintains a pair of shadow maps.  One is used to track the
state of memory in the kernel map at bit-granularity: a bit in the
kernel map is initialized when the corresponding shadow bit is clear,
and is uninitialized otherwise.  The second shadow map stores
information about the origin of uninitialized regions of the kernel map,
simplifying debugging.

KMSAN relies on being able to intercept certain functions which cannot
be instrumented by the compiler.  KMSAN thus implements interceptors
which manually update shadow state and in some cases explicitly check
for uninitialized bytes.  For instance, all calls to copyout() are
subject to such checks.

The runtime exports several functions which can be used to verify the
shadow map for a given buffer.  Helpers provide the same functionality
for a few structures commonly used for I/O, such as CAM CCBs, BIOs and
mbufs.  These are handy when debugging a KMSAN report whose
proximate and root causes are far away from each other.

Obtained from:	NetBSD
Sponsored by:	The FreeBSD Foundation

sys/amd64/include/msan.h

@@ -0,0 +1,84 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 The FreeBSD Foundation
+ *
+ * This software was developed by Mark Johnston under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_MSAN_H_
+#define	_MACHINE_MSAN_H_
+
+#ifdef KMSAN
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <machine/vmparam.h>
+
+typedef uint32_t msan_orig_t;
+
+/*
+ * Our 32-bit origin cells encode a 2-bit type and 30-bit pointer.  The pointer
+ * is compressed by making it a positive offset relative to KERNBASE.
+ */
+#define	KMSAN_ORIG_TYPE_SHIFT	30u
+#define	KMSAN_ORIG_PTR_MASK	((1u << KMSAN_ORIG_TYPE_SHIFT) - 1)
+
+static inline msan_orig_t
+kmsan_md_orig_encode(int type, uintptr_t ptr)
+{
+	return ((type << KMSAN_ORIG_TYPE_SHIFT) |
+	    ((ptr & KMSAN_ORIG_PTR_MASK)));
+}
+
+static inline void
+kmsan_md_orig_decode(msan_orig_t orig, int *type, uintptr_t *ptr)
+{
+	*type = orig >> KMSAN_ORIG_TYPE_SHIFT;
+	*ptr = (orig & KMSAN_ORIG_PTR_MASK) | KERNBASE;
+}
+
+static inline vm_offset_t
+kmsan_md_addr_to_shad(vm_offset_t addr)
+{
+	return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_SHAD_MIN_ADDRESS);
+}
+
+static inline vm_offset_t
+kmsan_md_addr_to_orig(vm_offset_t addr)
+{
+	return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_ORIG_MIN_ADDRESS);
+}
+
+static inline bool
+kmsan_md_unsupported(vm_offset_t addr)
+{
+	return (addr < VM_MIN_KERNEL_ADDRESS || addr >= KERNBASE);
+}
+
+#endif /* KMSAN */
+
+#endif /* !_MACHINE_MSAN_H_ */

sys/conf/files

@@ -3925,6 +3925,8 @@ kern/subr_lock.c		standard
 kern/subr_log.c			standard
 kern/subr_mchain.c		optional libmchain
 kern/subr_module.c		standard
+kern/subr_msan.c		optional kmsan \
+	compile-with "${NORMAL_C:N-fsanitize*}"
 kern/subr_msgbuf.c		standard
 kern/subr_param.c		standard
 kern/subr_pcpu.c		standard

sys/kern/kern_thread.c

@@ -84,11 +84,11 @@ __FBSDID("$FreeBSD$");
  * structures.
  */
 #ifdef __amd64__
-_Static_assert(offsetof(struct thread, td_flags) == 0xfc,
+_Static_assert(offsetof(struct thread, td_flags) == 0x108,
     "struct thread KBI td_flags");
-_Static_assert(offsetof(struct thread, td_pflags) == 0x104,
+_Static_assert(offsetof(struct thread, td_pflags) == 0x110,
     "struct thread KBI td_pflags");
-_Static_assert(offsetof(struct thread, td_frame) == 0x4a0,
+_Static_assert(offsetof(struct thread, td_frame) == 0x4a8,
     "struct thread KBI td_frame");
 _Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0,
     "struct thread KBI td_emuldata");
@@ -104,13 +104,13 @@ _Static_assert(offsetof(struct proc, p_emuldata) == 0x4b8,
     "struct proc KBI p_emuldata");
 #endif
 #ifdef __i386__
-_Static_assert(offsetof(struct thread, td_flags) == 0x98,
+_Static_assert(offsetof(struct thread, td_flags) == 0x9c,
     "struct thread KBI td_flags");
-_Static_assert(offsetof(struct thread, td_pflags) == 0xa0,
+_Static_assert(offsetof(struct thread, td_pflags) == 0xa4,
     "struct thread KBI td_pflags");
-_Static_assert(offsetof(struct thread, td_frame) == 0x304,
+_Static_assert(offsetof(struct thread, td_frame) == 0x308,
     "struct thread KBI td_frame");
-_Static_assert(offsetof(struct thread, td_emuldata) == 0x348,
+_Static_assert(offsetof(struct thread, td_emuldata) == 0x34c,
     "struct thread KBI td_emuldata");
 _Static_assert(offsetof(struct proc, p_flag) == 0x6c,
     "struct proc KBI p_flag");

sys/sys/msan.h

@@ -0,0 +1,87 @@
+/*	$NetBSD: msan.h,v 1.2 2020/09/09 16:29:59 maxv Exp $	*/
+
+/*
+ * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
+ * All rights reserved.
+ *
+ * This code is part of the KMSAN subsystem of the NetBSD kernel.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SYS_MSAN_H_
+#define _SYS_MSAN_H_
+
+#ifdef KMSAN
+#include <sys/_bus_dma.h>
+#include <sys/types.h>
+
+#define KMSAN_STATE_UNINIT	0xFF
+#define KMSAN_STATE_INITED	0x00
+
+#define KMSAN_TYPE_STACK	0
+#define KMSAN_TYPE_KMEM		1
+#define KMSAN_TYPE_MALLOC	2
+#define KMSAN_TYPE_UMA		3
+#define KMSAN_TYPE_MAX		3
+
+#define KMSAN_RET_ADDR		(uintptr_t)__builtin_return_address(0)
+
+union ccb;
+struct bio;
+struct mbuf;
+struct memdesc;
+
+void kmsan_init(void);
+
+void kmsan_shadow_map(vm_offset_t, size_t);
+
+void kmsan_thread_alloc(struct thread *);
+void kmsan_thread_free(struct thread *);
+
+void kmsan_bus_dmamap_sync(struct memdesc *, bus_dmasync_op_t);
+
+void kmsan_orig(const void *, size_t, int, uintptr_t);
+void kmsan_mark(const void *, size_t, uint8_t);
+
+void kmsan_check(const void *, size_t, const char *);
+void kmsan_check_bio(const struct bio *, const char *);
+void kmsan_check_ccb(const union ccb *, const char *);
+void kmsan_check_mbuf(const struct mbuf *, const char *);
+
+#else
+#define kmsan_init(u)
+#define kmsan_shadow_map(a, s)
+#define kmsan_thread_alloc(td)
+#define kmsan_thread_free(l)
+#define kmsan_dma_sync(m, a, s, o)
+#define kmsan_dma_load(m, b, s, o)
+#define kmsan_orig(p, l, c, a)
+#define kmsan_mark(p, l, c)
+#define kmsan_check(b, s, d)
+#define kmsan_check_bio(b, d)
+#define kmsan_check_ccb(c, d)
+#define kmsan_check_mbuf(m, d)
+#define	kmsan_bus_dmamap_sync(d, op)
+#endif
+
+#endif /* !_SYS_MSAN_H_ */

sys/sys/proc.h

@@ -183,6 +183,7 @@ struct kaudit_record;
 struct kcov_info;
 struct kdtrace_proc;
 struct kdtrace_thread;
+struct kmsan_td;
 struct kq_timer_cb_data;
 struct mqueue_notifier;
 struct p_sched;
@@ -250,6 +251,7 @@ struct thread {
 #define	td_siglist	td_sigqueue.sq_signals
 	u_char		td_lend_user_pri; /* (t) Lend user pri. */
 	u_char		td_allocdomain;	/* (b) NUMA domain backing this struct thread. */
+	struct kmsan_td	*td_kmsan;	/* (k) KMSAN state */
 
 /* Cleared during fork1() */
 #define	td_startzero td_flags