mirror of
https://git.FreeBSD.org/src.git
synced 2024-12-22 11:17:19 +00:00
Implement atomic_fcmpset_* for arm and arm64.
This commit is contained in:
Olivier Houchard
parent
835809f99f
commit
dc5f9fcdae
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=312925
@ -112,6 +112,43 @@ atomic_clear_64(volatile uint64_t *address, uint64_t clearmask)
|
||||
__with_interrupts_disabled(*address &= ~clearmask);
|
||||
}
|
||||
|
||||
static __inline int
|
||||
atomic_fcmpset_32(volatile u_int32_t *p, volatile u_int32_t *cmpval, volatile u_int32_t newval)
|
||||
{
|
||||
u_int32_t ret;
|
||||
|
||||
__with_interrupts_disabled(
|
||||
{
|
||||
ret = *p;
|
||||
if (*p == *cmpval) {
|
||||
*p = newval;
|
||||
ret = 1;
|
||||
} else {
|
||||
*cmpval = *p;
|
||||
ret = 0;
|
||||
}
|
||||
});
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static __inline int
|
||||
atomic_fcmpset_64(volatile u_int64_t *p, volatile u_int64_t *cmpval, volatile u_int64_t newval)
|
||||
{
|
||||
u_int64_t ret;
|
||||
|
||||
__with_interrupts_disabled(
|
||||
{
|
||||
if (*p == *cmpval) {
|
||||
*p = newval;
|
||||
ret = 1;
|
||||
} else {
|
||||
*cmpval = *p;
|
||||
ret = 0;
|
||||
}
|
||||
});
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static __inline u_int32_t
|
||||
atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
|
||||
{
|
||||
@ -370,6 +407,12 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
|
||||
return (__swp(v, p));
|
||||
}
|
||||
|
||||
#define atomic_fcmpset_rel_32 atomic_fcmpset_32
|
||||
#define atomic_fcmpset_acq_32 atomic_fcmpset_32
|
||||
#define atomic_fcmpset_rel_64 atomic_fcmpset_64
|
||||
#define atomic_fcmpset_acq_64 atomic_fcmpset_64
|
||||
#define atomic_fcmpset_acq_long atomic_fcmpset_long
|
||||
#define atomic_fcmpset_rel_long atomic_fcmpset_long
|
||||
#define atomic_cmpset_rel_32 atomic_cmpset_32
|
||||
#define atomic_cmpset_acq_32 atomic_cmpset_32
|
||||
#define atomic_cmpset_rel_64 atomic_cmpset_64
|
||||
@ -420,6 +463,14 @@ atomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe)
|
||||
return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe));
|
||||
}
|
||||
|
||||
static __inline u_long
|
||||
atomic_fcmpset_long(volatile u_long *dst, u_long *old, u_long newe)
|
||||
{
|
||||
|
||||
return (atomic_fcmpset_32((volatile uint32_t *)dst,
|
||||
(uint32_t *)old, newe));
|
||||
}
|
||||
|
||||
static __inline u_long
|
||||
atomic_fetchadd_long(volatile u_long *p, u_long v)
|
||||
{
|
||||
|
||||
@ -190,6 +190,116 @@ ATOMIC_ACQ_REL(clear, 32)
|
||||
ATOMIC_ACQ_REL(clear, 64)
|
||||
ATOMIC_ACQ_REL_LONG(clear)
|
||||
|
||||
static __inline int
|
||||
atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
|
||||
{
|
||||
uint32_t tmp;
|
||||
uint32_t _cmpval = *cmpval;
|
||||
int ret;
|
||||
|
||||
__asm __volatile(
|
||||
"1: mov %0, #1 \n"
|
||||
" ldrex %1, [%2] \n"
|
||||
" cmp %1, %3 \n"
|
||||
" it ne \n"
|
||||
" bne 2f \n"
|
||||
" strex %0, %4, [%2] \n"
|
||||
"2:"
|
||||
: "=&r" (ret), "=&r" (tmp), "+r" (p), "+r" (_cmpval), "+r" (newval)
|
||||
: : "cc", "memory");
|
||||
*cmpval = tmp;
|
||||
return (!ret);
|
||||
}
|
||||
|
||||
static __inline uint64_t
|
||||
atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
|
||||
{
|
||||
uint64_t tmp;
|
||||
uint64_t _cmpval = *cmpval;
|
||||
int ret;
|
||||
|
||||
__asm __volatile(
|
||||
"1: mov %[ret], #1 \n"
|
||||
" ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n"
|
||||
" teq %Q[tmp], %Q[_cmpval] \n"
|
||||
" itee eq \n"
|
||||
" teqeq %R[tmp], %R[_cmpval] \n"
|
||||
" bne 2f \n"
|
||||
" strexd %[ret], %Q[newval], %R[newval], [%[ptr]]\n"
|
||||
"2: \n"
|
||||
: [ret] "=&r" (ret),
|
||||
[tmp] "=&r" (tmp)
|
||||
: [ptr] "r" (p),
|
||||
[_cmpval] "r" (_cmpval),
|
||||
[newval] "r" (newval)
|
||||
: "cc", "memory");
|
||||
*cmpval = tmp;
|
||||
return (!ret);
|
||||
}
|
||||
|
||||
static __inline u_long
|
||||
atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
|
||||
{
|
||||
|
||||
return (atomic_fcmpset_32((volatile uint32_t *)p,
|
||||
(uint32_t *)cmpval, newval));
|
||||
}
|
||||
|
||||
static __inline uint64_t
|
||||
atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
|
||||
{
|
||||
uint64_t ret;
|
||||
|
||||
ret = atomic_fcmpset_64(p, cmpval, newval);
|
||||
dmb();
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static __inline u_long
|
||||
atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
|
||||
{
|
||||
u_long ret;
|
||||
|
||||
ret = atomic_fcmpset_long(p, cmpval, newval);
|
||||
dmb();
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static __inline uint32_t
|
||||
atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
|
||||
{
|
||||
|
||||
uint32_t ret;
|
||||
|
||||
ret = atomic_fcmpset_32(p, cmpval, newval);
|
||||
dmb();
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static __inline uint32_t
|
||||
atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
|
||||
{
|
||||
|
||||
dmb();
|
||||
return (atomic_fcmpset_32(p, cmpval, newval));
|
||||
}
|
||||
|
||||
static __inline uint64_t
|
||||
atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
|
||||
{
|
||||
|
||||
dmb();
|
||||
return (atomic_fcmpset_64(p, cmpval, newval));
|
||||
}
|
||||
|
||||
static __inline u_long
|
||||
atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
|
||||
{
|
||||
|
||||
dmb();
|
||||
return (atomic_fcmpset_long(p, cmpval, newval));
|
||||
}
|
||||
|
||||
static __inline uint32_t
|
||||
atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
|
||||
{
|
||||
|
||||
@ -84,6 +84,9 @@ atomic_store_long(volatile u_long *dst, u_long src)
|
||||
#define atomic_set_ptr atomic_set_32
|
||||
#define atomic_set_acq_ptr atomic_set_acq_32
|
||||
#define atomic_set_rel_ptr atomic_set_rel_32
|
||||
#define atomic_fcmpset_ptr atomic_fcmpset_32
|
||||
#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_32
|
||||
#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_32
|
||||
#define atomic_cmpset_ptr atomic_cmpset_32
|
||||
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32
|
||||
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32
|
||||
@ -105,6 +108,9 @@ atomic_store_long(volatile u_long *dst, u_long src)
|
||||
#define atomic_set_int atomic_set_32
|
||||
#define atomic_set_acq_int atomic_set_acq_32
|
||||
#define atomic_set_rel_int atomic_set_rel_32
|
||||
#define atomic_fcmpset_int atomic_fcmpset_32
|
||||
#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
|
||||
#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
|
||||
#define atomic_cmpset_int atomic_cmpset_32
|
||||
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
|
||||
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
|
||||
|
||||
@ -98,6 +98,61 @@ ATOMIC(clear, bic)
|
||||
ATOMIC(set, orr)
|
||||
ATOMIC(subtract, sub)
|
||||
|
||||
#define ATOMIC_FCMPSET(bar, a, l) \
|
||||
static __inline int \
|
||||
atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \
|
||||
uint32_t newval) \
|
||||
{ \
|
||||
uint32_t tmp; \
|
||||
uint32_t _cmpval = *cmpval; \
|
||||
int res; \
|
||||
\
|
||||
__asm __volatile( \
|
||||
"1: mov %w1, #1 \n" \
|
||||
" ld"#a"xr %w0, [%2] \n" \
|
||||
" cmp %w0, %w3 \n" \
|
||||
" b.ne 2f \n" \
|
||||
" st"#l"xr %w1, %w4, [%2] \n" \
|
||||
"2:" \
|
||||
: "=&r"(tmp), "=&r"(res) \
|
||||
: "r" (p), "r" (_cmpval), "r" (newval) \
|
||||
: "cc", "memory" \
|
||||
); \
|
||||
*cmpval = tmp; \
|
||||
\
|
||||
return (!res); \
|
||||
} \
|
||||
\
|
||||
static __inline int \
|
||||
atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \
|
||||
uint64_t newval) \
|
||||
{ \
|
||||
uint64_t tmp; \
|
||||
uint64_t _cmpval = *cmpval; \
|
||||
int res; \
|
||||
\
|
||||
__asm __volatile( \
|
||||
"1: mov %w1, #1 \n" \
|
||||
" ld"#a"xr %0, [%2] \n" \
|
||||
" cmp %0, %3 \n" \
|
||||
" b.ne 2f \n" \
|
||||
" st"#l"xr %w1, %4, [%2] \n" \
|
||||
"2:" \
|
||||
: "=&r"(tmp), "=&r"(res) \
|
||||
: "r" (p), "r" (_cmpval), "r" (newval) \
|
||||
: "cc", "memory" \
|
||||
); \
|
||||
*cmpval = tmp; \
|
||||
\
|
||||
return (!res); \
|
||||
}
|
||||
|
||||
ATOMIC_FCMPSET( , , )
|
||||
ATOMIC_FCMPSET(acq_, a, )
|
||||
ATOMIC_FCMPSET(rel_, ,l)
|
||||
|
||||
#undef ATOMIC_FCMPSET
|
||||
|
||||
#define ATOMIC_CMPSET(bar, a, l) \
|
||||
static __inline int \
|
||||
atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \
|
||||
@ -311,6 +366,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
|
||||
|
||||
#define atomic_add_int atomic_add_32
|
||||
#define atomic_fcmpset_int atomic_fcmpset_32
|
||||
#define atomic_clear_int atomic_clear_32
|
||||
#define atomic_cmpset_int atomic_cmpset_32
|
||||
#define atomic_fetchadd_int atomic_fetchadd_32
|
||||
@ -320,6 +376,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_int atomic_subtract_32
|
||||
|
||||
#define atomic_add_acq_int atomic_add_acq_32
|
||||
#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
|
||||
#define atomic_clear_acq_int atomic_clear_acq_32
|
||||
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
|
||||
#define atomic_load_acq_int atomic_load_acq_32
|
||||
@ -327,6 +384,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_acq_int atomic_subtract_acq_32
|
||||
|
||||
#define atomic_add_rel_int atomic_add_rel_32
|
||||
#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
|
||||
#define atomic_clear_rel_int atomic_add_rel_32
|
||||
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
|
||||
#define atomic_set_rel_int atomic_set_rel_32
|
||||
@ -334,6 +392,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_store_rel_int atomic_store_rel_32
|
||||
|
||||
#define atomic_add_long atomic_add_64
|
||||
#define atomic_fcmpset_long atomic_fcmpset_64
|
||||
#define atomic_clear_long atomic_clear_64
|
||||
#define atomic_cmpset_long atomic_cmpset_64
|
||||
#define atomic_fetchadd_long atomic_fetchadd_64
|
||||
@ -343,6 +402,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_long atomic_subtract_64
|
||||
|
||||
#define atomic_add_ptr atomic_add_64
|
||||
#define atomic_fcmpset_ptr atomic_fcmpset_64
|
||||
#define atomic_clear_ptr atomic_clear_64
|
||||
#define atomic_cmpset_ptr atomic_cmpset_64
|
||||
#define atomic_fetchadd_ptr atomic_fetchadd_64
|
||||
@ -352,6 +412,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_ptr atomic_subtract_64
|
||||
|
||||
#define atomic_add_acq_long atomic_add_acq_64
|
||||
#define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
|
||||
#define atomic_clear_acq_long atomic_add_acq_64
|
||||
#define atomic_cmpset_acq_long atomic_cmpset_acq_64
|
||||
#define atomic_load_acq_long atomic_load_acq_64
|
||||
@ -359,6 +420,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_acq_long atomic_subtract_acq_64
|
||||
|
||||
#define atomic_add_acq_ptr atomic_add_acq_64
|
||||
#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
|
||||
#define atomic_clear_acq_ptr atomic_add_acq_64
|
||||
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
|
||||
#define atomic_load_acq_ptr atomic_load_acq_64
|
||||
@ -366,6 +428,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_subtract_acq_ptr atomic_subtract_acq_64
|
||||
|
||||
#define atomic_add_rel_long atomic_add_rel_64
|
||||
#define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
|
||||
#define atomic_clear_rel_long atomic_clear_rel_64
|
||||
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
|
||||
#define atomic_set_rel_long atomic_set_rel_64
|
||||
@ -373,6 +436,7 @@ atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
#define atomic_store_rel_long atomic_store_rel_64
|
||||
|
||||
#define atomic_add_rel_ptr atomic_add_rel_64
|
||||
#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
|
||||
#define atomic_clear_rel_ptr atomic_clear_rel_64
|
||||
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
|
||||
#define atomic_set_rel_ptr atomic_set_rel_64
|
||||
|
||||
Loading…
Reference in New Issue
Block a user