From 6443773dab50613cf44754f6f0a5f587fa11b1d4 Mon Sep 17 00:00:00 2001 From: Matt Macy Date: Tue, 3 Jul 2018 01:55:09 +0000 Subject: [PATCH] make critical_{enter, exit} inline Avoid pulling in all of the dependencies by automatically generating a stripped down thread_lite exporting only the fields of interest. The field declarations are type checked against the original and the offsets of the generated result is automatically checked. kib has expressed disagreement and would have preferred to simply use genassym style offsets (which loses type check enforcement). jhb has expressed dislike of it due to header pollution and a duplicate structure. He would have preferred to just have defined thread in _thread.h. Nonetheless, he admits that this is the only viable solution at the moment. The impetus for this came from mjg's D15331: "Inline critical_enter/exit for amd64" Reviewed by: jeff Differential Revision: https://reviews.freebsd.org/D16078 --- sys/conf/kern.post.mk | 22 +++++-- sys/conf/kern.pre.mk | 2 +- sys/kern/genoffset.c | 44 +++++++++++++ sys/kern/genoffset.sh | 141 +++++++++++++++++++++++++++++++++++++++++ sys/kern/kern_switch.c | 27 ++++---- sys/sys/assym.h | 15 +++++ sys/sys/kpilite.h | 55 ++++++++++++++++ sys/sys/systm.h | 42 +++++++++++- 8 files changed, 324 insertions(+), 24 deletions(-) create mode 100644 sys/kern/genoffset.c create mode 100644 sys/kern/genoffset.sh create mode 100644 sys/sys/kpilite.h diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk index f8d21c217e35..24c1db2c3dc3 100644 --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -185,13 +185,25 @@ hack.pico: Makefile ${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico rm -f hack.c -assym.inc: $S/kern/genassym.sh genassym.o +offset.inc: $S/kern/genoffset.sh genoffset.o + NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genoffset.sh genoffset.o > ${.TARGET} + +genoffset.o: $S/kern/genoffset.c + ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/kern/genoffset.c + +genoffset_test.c: $S/kern/genoffset.c + cp $S/kern/genoffset.c genoffset_test.c + +genoffset_test.o: genoffset_test.c offset.inc + ${CC} -c ${CFLAGS:N-flto:N-fno-common} -DOFFSET_TEST genoffset_test.c + +assym.inc: $S/kern/genassym.sh genassym.o genoffset_test.o NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET} -genassym.o: $S/$M/$M/genassym.c +genassym.o: $S/$M/$M/genassym.c offset.inc ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c -${SYSTEM_OBJS} genassym.o vers.o: opt_global.h +${SYSTEM_OBJS} genoffset.o genassym.o vers.o: opt_global.h .if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon) _meta_filemon= 1 @@ -213,10 +225,10 @@ _SKIP_DEPEND= 1 .endif kernel-depend: .depend -SRCS= assym.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ +SRCS= assym.inc offset.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ ${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \ ${MFILES:T:S/.m$/.h/} -DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o +DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o genoffset.o DEPENDFILES= ${DEPENDOBJS:O:u:C/^/.depend./} .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk index c1bbf0d30bfb..92f0fe968ce6 100644 --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -195,7 +195,7 @@ OFEDCFLAGS= ${CFLAGS:N-I*} -DCONFIG_INFINIBAND_USER_MEM \ OFED_C_NOIMP= ${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF} OFED_C= ${OFED_C_NOIMP} ${.IMPSRC} -GEN_CFILES= $S/$M/$M/genassym.c ${MFILES:T:S/.m$/.c/} +GEN_CFILES= $S/$M/$M/genassym.c $S/kern/genoffset.c ${MFILES:T:S/.m$/.c/} SYSTEM_CFILES= config.c env.c hints.c vnode_if.c SYSTEM_DEP= Makefile ${SYSTEM_OBJS} SYSTEM_OBJS= locore.o ${MDOBJS} ${OBJS} diff --git a/sys/kern/genoffset.c b/sys/kern/genoffset.c new file mode 100644 index 000000000000..4a1e12a1d072 --- /dev/null +++ b/sys/kern/genoffset.c @@ -0,0 +1,44 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef OFFSET_TEST +#define GENOFFSET +#endif +#include +__FBSDID("$FreeBSD$"); +#include +#include +#include + +OFFSYM(td_pre_epoch_prio, thread, u_char); +OFFSYM(td_priority, thread, u_char); +OFFSYM(td_epochnest, thread, u_char); +OFFSYM(td_critnest, thread, u_int); +OFFSYM(td_pinned, thread, int); +OFFSYM(td_owepreempt, thread, u_char); diff --git a/sys/kern/genoffset.sh b/sys/kern/genoffset.sh new file mode 100644 index 000000000000..4318cf85bce9 --- /dev/null +++ b/sys/kern/genoffset.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +# SPDX-License-Identifier: BSD-2-Clause-FreeBSD +# +# Copyright (c) 2000, Bruce Evans +# Copyright (c) 2018, Jeff Roberson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +usage() +{ + echo "usage: genoffset [-o outfile] objfile" + exit 1 +} + + +work() +{ + echo "#ifndef _OFFSET_INC_" + echo "#define _OFFSET_INC_" + echo "#if !defined(GENOFFSET) && !defined(KLD_MODULE)" + ${NM:='nm'} ${NMFLAGS} "$1" | ${AWK:='awk'} ' + / C .*_datatype_*/ { + type = substr($3, match($3, "_datatype_") + length("_datatype_")) + } + / C .*_parenttype_*/ { + parent = substr($3, match($3, "_parenttype_") + length("_parenttype_")) + } + / C .*sign$/ { + sign = substr($1, length($1) - 3, 4) + sub("^0*", "", sign) + if (sign != "") + sign = "-" + } + / C .*w0$/ { + w0 = substr($1, length($1) - 3, 4) + } + / C .*w1$/ { + w1 = substr($1, length($1) - 3, 4) + } + / C .*w2$/ { + w2 = substr($1, length($1) - 3, 4) + } + / C .*w3$/ { + w3 = substr($1, length($1) - 3, 4) + w = w3 w2 w1 w0 + sub("^0*", "", w) + if (w == "") + w = "0" + hex = "" + if (w != "0") + hex = "0x" + sub("w3$", "", $3) + member = tolower($3) + # This still has minor problems representing INT_MIN, etc. + # E.g., + # with 32-bit 2''s complement ints, this prints -0x80000000, + # which has the wrong type (unsigned int). + offset = sprintf("%s%s%s", sign, hex, w) + + structures[parent] = sprintf("%s%s %s %s\n", + structures[parent], offset, type, member) + } + END { + for (struct in structures) { + printf("struct %s_lite {\n", struct); + n = split(structures[struct], members, "\n") + for (i = 1; i < n; i++) { + for (j = i + 1; j < n; j++) { + split(members[i], ivar, " ") + split(members[j], jvar, " ") + if (jvar[1] < ivar[1]) { + tmp = members[i] + members[i] = members[j] + members[j] = tmp + } + } + } + off = "0" + for (i = 1; i < n; i++) { + split(members[i], m, " ") + printf "\tu_char\tpad_%s[%s - %s];\n", m[3], m[1], off + printf "\t%s\t%s;\n", m[2], m[3] + off = sprintf("(%s + sizeof(%s))", m[1], m[2]) + } + printf("};\n"); + } + } + ' + + echo "#endif" + echo "#endif" +} + + +# +#MAIN PROGGRAM +# +use_outfile="no" +while getopts "o:" option +do + case "$option" in + o) outfile="$OPTARG" + use_outfile="yes";; + *) usage;; + esac +done +shift $(($OPTIND - 1)) +case $# in +1) ;; +*) usage;; +esac + +if [ "$use_outfile" = "yes" ] +then + work $1 3>"$outfile" >&3 3>&- +else + work $1 +fi + diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index ee7d65d8b261..254c4ffd2674 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -199,17 +199,17 @@ choosethread(void) * the function call itself, for most cases. */ void -critical_enter(void) +critical_enter_KBI(void) { - struct thread *td; - - td = curthread; - td->td_critnest++; +#ifdef KTR + struct thread *td = curthread; +#endif + critical_enter(); CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } -static void __noinline +void __noinline critical_exit_preempt(void) { struct thread *td; @@ -245,17 +245,12 @@ critical_exit_preempt(void) } void -critical_exit(void) +critical_exit_KBI(void) { - struct thread *td; - - td = curthread; - KASSERT(td->td_critnest != 0, - ("critical_exit: td_critnest == 0")); - td->td_critnest--; - __compiler_membar(); - if (__predict_false(td->td_owepreempt)) - critical_exit_preempt(); +#ifdef KTR + struct thread *td = curthread; +#endif + critical_exit(); CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } diff --git a/sys/sys/assym.h b/sys/sys/assym.h index 0ef1286bccb6..f5204fa83b16 100644 --- a/sys/sys/assym.h +++ b/sys/sys/assym.h @@ -43,4 +43,19 @@ char name ## w1[((ASSYM_ABS(value) & 0xFFFF0000UL) >> 16) + ASSYM_BIAS]; \ char name ## w2[((ASSYM_ABS(value) & 0xFFFF00000000ULL) >> 32) + ASSYM_BIAS]; \ char name ## w3[((ASSYM_ABS(value) & 0xFFFF000000000000ULL) >> 48) + ASSYM_BIAS] + +/* char name ## _datatype_ ## STRINGIFY(typeof(((struct parenttype *)(0x0))-> name)) [1]; */ +#ifdef OFFSET_TEST +#define OFFSET_CTASSERT CTASSERT +#else +#define OFFSET_CTASSERT(...) +#endif + +#define OFFSYM(name, parenttype, datatype) \ +ASSYM(name, offsetof(struct parenttype, name)); \ +char name ## _datatype_ ## datatype [1]; \ +char name ## _parenttype_ ## parenttype [1]; \ +CTASSERT(__builtin_types_compatible_p(__typeof(((struct parenttype *)(0x0))-> name), datatype)); \ +OFFSET_CTASSERT(offsetof(struct parenttype, name) == offsetof(struct parenttype ## _lite, name)) + #endif /* !_SYS_ASSYM_H_ */ diff --git a/sys/sys/kpilite.h b/sys/sys/kpilite.h new file mode 100644 index 000000000000..53817550a8f2 --- /dev/null +++ b/sys/sys/kpilite.h @@ -0,0 +1,55 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_KPILITE_H_ +#define _SYS_KPILITE_H_ +#if !defined(GENOFFSET) && !defined(KLD_MODULE) && defined(_KERNEL) +#include "offset.inc" + +static __inline void +sched_pin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_pin called on non curthread")); + td->td_pinned++; + __compiler_membar(); +} + +static __inline void +sched_unpin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_unpin called on non curthread")); + KASSERT(td->td_pinned > 0, ("sched_unpin called on non pinned thread")); + __compiler_membar(); + td->td_pinned--; + __compiler_membar(); +} +#endif +#endif diff --git a/sys/sys/systm.h b/sys/sys/systm.h index c8d32b4a00e2..776f10d5347d 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -107,6 +107,12 @@ void kassert_panic(const char *fmt, ...) __printflike(1, 2); #define CTASSERT(x) _Static_assert(x, "compile-time assertion failed") #endif +#if defined(_KERNEL) +#include /* MAXCPU */ +#include /* curthread */ +#include +#endif + /* * Assert that a pointer can be loaded from memory atomically. * @@ -214,12 +220,44 @@ void vpanic(const char *, __va_list) __dead2 __printflike(1, 0); void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); +void critical_enter_KBI(void); +void critical_exit_KBI(void); +void critical_exit_preempt(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); void tablefull(const char *); + +#if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET) +#define critical_enter() critical_enter_KBI() +#define critical_exit() critical_exit_KBI() +#else +static __inline void +critical_enter(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + td->td_critnest++; +} + +static __inline void +critical_exit(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); + td->td_critnest--; + __compiler_membar(); + if (__predict_false(td->td_owepreempt)) + critical_exit_preempt(); + +} +#endif + + #ifdef EARLY_PRINTF typedef void early_putc_t(int ch); extern early_putc_t *early_putc;