From 8a1ee2d346716fc0c13fa3349839ad1aed0144ad Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Wed, 21 Aug 2013 22:03:06 +0000 Subject: [PATCH] Implement atomic_swap() and atomic_testandset(). Reviewed by: arch, bde, jilles, kib --- share/man/man9/atomic.9 | 83 ++++++++++++++++++++++++++++++-------- sys/amd64/include/atomic.h | 72 ++++++++++++++++++++++++++------- sys/i386/include/atomic.h | 62 +++++++++++++++++++--------- 3 files changed, 166 insertions(+), 51 deletions(-) diff --git a/share/man/man9/atomic.9 b/share/man/man9/atomic.9 index 0baac4557002..a2c0a67975b3 100644 --- a/share/man/man9/atomic.9 +++ b/share/man/man9/atomic.9 @@ -23,7 +23,7 @@ .\" .\" $FreeBSD$ .\" -.Dd September 27, 2005 +.Dd August 20, 2013 .Dt ATOMIC 9 .Os .Sh NAME @@ -62,6 +62,10 @@ .Fn atomic_subtract_[acq_|rel_] "volatile *p" " v" .Ft void .Fn atomic_store_rel_ "volatile *p" " v" +.Ft +.Fn atomic_swap_ "volatile *p" " v" +.Ft int +.Fn atomic_testandset_ "volatile *p" "u_int v" .Sh DESCRIPTION Each of the atomic operations is guaranteed to be atomic in the presence of interrupts. @@ -184,9 +188,9 @@ This section describes the semantics of each operation using a C like notation. .Bd -literal -compact if (*dst == old) { *dst = new; - return 1; + return (1); } else - return 0; + return (0); .Ed .El .Pp @@ -203,7 +207,7 @@ and .Bd -literal -compact tmp = *p; *p += v; -return tmp; +return (tmp); .Ed .El .Pp @@ -216,9 +220,9 @@ and .Dq Li 32 and do not have any variants with memory barriers at this time. .Bl -hang -.It Fn atomic_load addr +.It Fn atomic_load p .Bd -literal -compact -return (*addr) +return (*p); .Ed .El .Pp @@ -226,11 +230,11 @@ The .Fn atomic_load functions are only provided with acquire memory barriers. .Bl -hang -.It Fn atomic_readandclear addr +.It Fn atomic_readandclear p .Bd -literal -compact -temp = *addr; -*addr = 0; -return (temp); +tmp = *p; +*p = 0; +return (tmp); .Ed .El .Pp @@ -243,8 +247,7 @@ functions are not implemented for the types .Dq Li 8 , and .Dq Li 16 -and do -not have any variants with memory barriers at this time. +and do not have any variants with memory barriers at this time. .Bl -hang .It Fn atomic_set p v .Bd -literal -compact @@ -264,6 +267,44 @@ The .Fn atomic_store functions are only provided with release memory barriers. .Pp +.Bl -hang +.It Fn atomic_swap p v +.Bd -literal -compact +tmp = *p; +*p = v; +return (tmp); +.Ed +.El +.Pp +The +.Fn atomic_swap +functions are not implemented for the types +.Dq Li char , +.Dq Li short , +.Dq Li ptr , +.Dq Li 8 , +and +.Dq Li 16 +and do not have any variants with memory barriers at this time. +.Bl -hang +.It Fn atomic_testandset p v +.Bd -literal -compact +bit = 1 << (v % (sizeof(*p) * NBBY)); +tmp = (*p & bit) != 0; +*p |= bit; +return (tmp); +.Ed +.El +.Pp +The +.Fn atomic_testandset +functions are only implemented for the types +.Dq Li int , +.Dq Li long +and +.Dq Li 32 +and do not have any variants with memory barriers at this time. +.Pp The type .Dq Li 64 is currently not implemented for any of the atomic operations on the @@ -275,15 +316,17 @@ architectures. .Sh RETURN VALUES The .Fn atomic_cmpset -function -returns the result of the compare operation. +function returns the result of the compare operation. The .Fn atomic_fetchadd , .Fn atomic_load , +.Fn atomic_readandclear , and -.Fn atomic_readandclear -functions -return the value at the specified address. +.Fn atomic_swap +functions return the value at the specified address. +The +.Fn atomic_testandset +function returns the result of the test operation. .Sh EXAMPLES This example uses the .Fn atomic_cmpset_acq_ptr @@ -354,3 +397,9 @@ The .Fn atomic_fetchadd operations were added in .Fx 6.0 . +The +.Fn atomic_swap +and +.Fn atomic_testandset +operations were added in +.Fx 10.0 . diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 7d8d839654be..4a15220f2e4c 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -54,12 +54,14 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -80,6 +82,8 @@ int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); +int atomic_testandset_int(volatile u_int *p, u_int v); +int atomic_testandset_long(volatile u_long *p, u_int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -205,6 +209,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Ir" (v & 0x1f) /* 2 */ + : "cc"); + return (res); +} + +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsq %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_long" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Jr" ((u_long)(v & 0x3f)) /* 2 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -296,43 +334,39 @@ ATOMIC_STORE(long); #ifndef WANT_FUNCTIONS -/* Read the current value and store a zero in the destination. */ +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *p) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ + "# atomic_swap_int" + : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ - return (res); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *p) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - res = 0; __asm __volatile( " xchgq %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ + "# atomic_swap_long" + : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ - return (res); + return (v); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *p); -u_long atomic_readandclear_long(volatile u_long *p); +u_int atomic_swap_int(volatile u_int *p, u_int v); +u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ @@ -376,6 +410,9 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -426,8 +463,10 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long @@ -447,7 +486,9 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long +#define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long +#define atomic_testandset_64 atomic_testandset_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long @@ -467,6 +508,7 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index f7822950435d..0d1738c66e97 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -54,12 +54,14 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -78,6 +80,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); +int atomic_testandset_int(volatile u_int *p, u_int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -275,6 +278,23 @@ atomic_fetchadd_int(volatile u_int *p, u_int v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Ir" (v & 0x1f) /* 2 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -386,43 +406,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } -/* Read the current value and store a zero in the destination. */ +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + + return (atomic_testandset_int((volatile u_int *)p, v)); +} + +/* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int -atomic_readandclear_int(volatile u_int *p) +atomic_swap_int(volatile u_int *p, u_int v) { - u_int res; - res = 0; __asm __volatile( " xchgl %1,%0 ; " - "# atomic_readandclear_int" - : "+r" (res), /* 0 */ + "# atomic_swap_int" + : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ - return (res); + return (v); } static __inline u_long -atomic_readandclear_long(volatile u_long *p) +atomic_swap_long(volatile u_long *p, u_long v) { - u_long res; - res = 0; - __asm __volatile( - " xchgl %1,%0 ; " - "# atomic_readandclear_long" - : "+r" (res), /* 0 */ - "+m" (*p)); /* 1 */ - - return (res); + return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #else /* !__GNUCLIKE_ASM */ -u_int atomic_readandclear_int(volatile u_int *p); -u_long atomic_readandclear_long(volatile u_long *p); +u_int atomic_swap_int(volatile u_int *p, u_int v); +u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ @@ -466,6 +483,9 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_readandclear_int(p) atomic_swap_int(p, 0) +#define atomic_readandclear_long(p) atomic_swap_long(p, 0) + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char @@ -516,8 +536,10 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ @@ -556,6 +578,8 @@ u_long atomic_readandclear_long(volatile u_long *p); #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) +#define atomic_swap_ptr(p, v) \ + atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p))