From 08c40841d8abaa433fe1bda5a320e44fe4e77445 Mon Sep 17 00:00:00 2001 From: Alan Cox Date: Wed, 18 Aug 1999 04:08:31 +0000 Subject: [PATCH] Create callable (non-inline) versions of the atomic_OP_TYPE functions that are linked into the kernel. The KLD compilation options are changed to call these functions, rather than in-lining the atomic operations. This approach makes atomic operations from KLDs significantly faster on UP systems (though somewhat slower on SMP systems). PR: i386/13111 Submitted by: peter.jeremy@alcatel.com.au --- sys/amd64/amd64/atomic.c | 47 ++++++++++++++++++++++++++++++++++++++ sys/amd64/include/atomic.h | 17 +++++++++++--- sys/conf/files.i386 | 4 +++- sys/i386/conf/files.i386 | 4 +++- sys/i386/i386/atomic.c | 47 ++++++++++++++++++++++++++++++++++++++ sys/i386/include/atomic.h | 17 +++++++++++--- 6 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 sys/amd64/amd64/atomic.c create mode 100644 sys/i386/i386/atomic.c diff --git a/sys/amd64/amd64/atomic.c b/sys/amd64/amd64/atomic.c new file mode 100644 index 00000000000..2c0443e7020 --- /dev/null +++ b/sys/amd64/amd64/atomic.c @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 1999 Peter Jeremy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id $ + */ + +/* This file creates publically callable functions to perform various + * simple arithmetic on memory which is atomic in the presence of + * interrupts and multiple processors. + */ +#include + +/* Firstly make atomic.h generate prototypes as it will for kernel modules */ +#define KLD_MODULE +#include +#undef _MACHINE_ATOMIC_H_ /* forget we included it */ +#undef KLD_MODULE +#undef ATOMIC_ASM + +/* Make atomic.h generate public functions */ +#define static +#undef __inline +#define __inline + +#include diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index d4e371bdebd..e9f9403f619 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: atomic.h,v 1.3 1999/07/13 06:35:25 alc Exp $ + * $Id: atomic.h,v 1.4 1999/07/23 23:45:19 alc Exp $ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ @@ -54,9 +54,19 @@ */ /* - * Make kernel modules portable between UP and SMP. + * The above functions are expanded inline in the statically-linked + * kernel. Lock prefixes are generated if an SMP kernel is being + * built. + * + * Kernel modules call real functions which are built into the kernel. + * This allows kernel modules to be portable between UP and SMP systems. */ -#if defined(SMP) || defined(KLD_MODULE) +#if defined(KLD_MODULE) +#define ATOMIC_ASM(NAME, TYPE, OP, V) \ + extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); + +#else /* !KLD_MODULE */ +#if defined(SMP) #define MPLOCKED "lock ; " #else #define MPLOCKED @@ -74,6 +84,7 @@ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ : "=m" (*p) \ : "0" (*p), "ir" (V)); \ } +#endif /* KLD_MODULE */ ATOMIC_ASM(set, char, "orb %2,%0", v) ATOMIC_ASM(clear, char, "andb %2,%0", ~v) diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index cf586fcc278..62e32697c32 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -1,7 +1,7 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # -# $Id: files.i386,v 1.257 1999/08/07 12:19:41 peter Exp $ +# $Id: files.i386,v 1.258 1999/08/09 10:34:38 phk Exp $ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and @@ -117,6 +117,8 @@ i386/eisa/dpt_eisa.c optional eisa dpt i386/eisa/eisaconf.c optional eisa i386/eisa/if_fea.c optional fea i386/eisa/if_vx_eisa.c optional vx +i386/i386/atomic.c standard \ + compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} $<" i386/i386/autoconf.c standard i386/i386/bios.c standard i386/i386/bioscall.s standard diff --git a/sys/i386/conf/files.i386 b/sys/i386/conf/files.i386 index cf586fcc278..62e32697c32 100644 --- a/sys/i386/conf/files.i386 +++ b/sys/i386/conf/files.i386 @@ -1,7 +1,7 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # -# $Id: files.i386,v 1.257 1999/08/07 12:19:41 peter Exp $ +# $Id: files.i386,v 1.258 1999/08/09 10:34:38 phk Exp $ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and @@ -117,6 +117,8 @@ i386/eisa/dpt_eisa.c optional eisa dpt i386/eisa/eisaconf.c optional eisa i386/eisa/if_fea.c optional fea i386/eisa/if_vx_eisa.c optional vx +i386/i386/atomic.c standard \ + compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} $<" i386/i386/autoconf.c standard i386/i386/bios.c standard i386/i386/bioscall.s standard diff --git a/sys/i386/i386/atomic.c b/sys/i386/i386/atomic.c new file mode 100644 index 00000000000..2c0443e7020 --- /dev/null +++ b/sys/i386/i386/atomic.c @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 1999 Peter Jeremy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id $ + */ + +/* This file creates publically callable functions to perform various + * simple arithmetic on memory which is atomic in the presence of + * interrupts and multiple processors. + */ +#include + +/* Firstly make atomic.h generate prototypes as it will for kernel modules */ +#define KLD_MODULE +#include +#undef _MACHINE_ATOMIC_H_ /* forget we included it */ +#undef KLD_MODULE +#undef ATOMIC_ASM + +/* Make atomic.h generate public functions */ +#define static +#undef __inline +#define __inline + +#include diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index d4e371bdebd..e9f9403f619 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: atomic.h,v 1.3 1999/07/13 06:35:25 alc Exp $ + * $Id: atomic.h,v 1.4 1999/07/23 23:45:19 alc Exp $ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ @@ -54,9 +54,19 @@ */ /* - * Make kernel modules portable between UP and SMP. + * The above functions are expanded inline in the statically-linked + * kernel. Lock prefixes are generated if an SMP kernel is being + * built. + * + * Kernel modules call real functions which are built into the kernel. + * This allows kernel modules to be portable between UP and SMP systems. */ -#if defined(SMP) || defined(KLD_MODULE) +#if defined(KLD_MODULE) +#define ATOMIC_ASM(NAME, TYPE, OP, V) \ + extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); + +#else /* !KLD_MODULE */ +#if defined(SMP) #define MPLOCKED "lock ; " #else #define MPLOCKED @@ -74,6 +84,7 @@ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ : "=m" (*p) \ : "0" (*p), "ir" (V)); \ } +#endif /* KLD_MODULE */ ATOMIC_ASM(set, char, "orb %2,%0", v) ATOMIC_ASM(clear, char, "andb %2,%0", ~v)