mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-12-27 11:55:06 +00:00
Code Issues Releases Activity
f33a947b56
freebsd/sys/kern/kern_lock.c
Attilio Rao 651175c9db Introduce support for adaptive spinning in lockmgr. 
Actually, as it did receive few tuning, the support is disabled by
default, but it can opt-in with the option ADAPTIVE_LOCKMGRS.
Due to the nature of lockmgrs, adaptive spinning needs to be
selectively enabled for any interested lockmgr.
The support is bi-directional, or, in other ways, it will work in both
cases if the lock is held in read or write way.  In particular, the
read path is passible of further tunning using the sysctls
debug.lockmgr.retries and debug.lockmgr.loops .  Ideally, such sysctls
should be axed or compiled out before release.

Addictionally note that adaptive spinning doesn't cope well with
LK_SLEEPFAIL.  The reason is that many (and probabilly all) consumers
of LK_SLEEPFAIL are mainly interested in knowing if the interlock was
dropped or not in order to reacquire it and re-test initial conditions.
This directly interacts with adaptive spinning because lockmgr needs
to drop the interlock while spinning in order to avoid a deadlock
(further details in the comments inside the patch).

Final note: finding someone willing to help on tuning this with
relevant workloads would be either very important and appreciated.

Tested by:	jeff, pho
Requested by:	many
2009-06-17 01:55:42 +00:00

1285 lines
34 KiB
C
Raw Blame History

/*-
* Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice(s), this list of conditions and the following disclaimer as
* the first lines of this file unmodified other than the possible
* addition of one or more copyright notices.
* 2. Redistributions in binary form must reproduce the above copyright
* notice(s), this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
#include "opt_adaptive_lockmgrs.h"
#include "opt_ddb.h"
#include "opt_kdtrace.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/ktr.h>
#include <sys/linker_set.h>
#include <sys/lock.h>
#include <sys/lock_profile.h>
#include <sys/lockmgr.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sleepqueue.h>
#ifdef DEBUG_LOCKS
#include <sys/stack.h>
#endif
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
(LK_ADAPTIVE | LK_NOSHARE));
CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
#define SQ_EXCLUSIVE_QUEUE 0
#define SQ_SHARED_QUEUE 1
#ifndef INVARIANTS
#define _lockmgr_assert(lk, what, file, line)
#define TD_LOCKS_INC(td)
#define TD_LOCKS_DEC(td)
#else
#define TD_LOCKS_INC(td) ((td)->td_locks++)
#define TD_LOCKS_DEC(td) ((td)->td_locks--)
#endif
#define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++)
#define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--)
#ifndef DEBUG_LOCKS
#define STACK_PRINT(lk)
#define STACK_SAVE(lk)
#define STACK_ZERO(lk)
#else
#define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
#define STACK_SAVE(lk) stack_save(&(lk)->lk_stack)
#define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack)
#endif
#define LOCK_LOG2(lk, string, arg1, arg2) \
if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
CTR2(KTR_LOCK, (string), (arg1), (arg2))
#define LOCK_LOG3(lk, string, arg1, arg2, arg3) \
if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
#define GIANT_DECLARE \
int _i = 0; \
WITNESS_SAVE_DECL(Giant)
#define GIANT_RESTORE() do { \
if (_i > 0) { \
while (_i--) \
mtx_lock(&Giant); \
WITNESS_RESTORE(&Giant.lock_object, Giant); \
} \
} while (0)
#define GIANT_SAVE() do { \
if (mtx_owned(&Giant)) { \
WITNESS_SAVE(&Giant.lock_object, Giant); \
while (mtx_owned(&Giant)) { \
_i++; \
mtx_unlock(&Giant); \
} \
} \
} while (0)
#define LK_CAN_SHARE(x) \
(((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 || \
((x) & LK_EXCLUSIVE_SPINNERS) == 0 || \
curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT)))
#define LK_TRYOP(x) \
((x) & LK_NOWAIT)
#define LK_CAN_WITNESS(x) \
(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
#define LK_TRYWIT(x) \
(LK_TRYOP(x) ? LOP_TRYLOCK : 0)
#define LK_CAN_ADAPT(lk, f) \
(((lk)->lock_object.lo_flags & LK_ADAPTIVE) != 0 && \
((f) & LK_SLEEPFAIL) == 0)
#define lockmgr_disowned(lk) \
(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
#define lockmgr_xlocked(lk) \
(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
static void assert_lockmgr(struct lock_object *lock, int how);
#ifdef DDB
static void db_show_lockmgr(struct lock_object *lock);
#endif
static void lock_lockmgr(struct lock_object *lock, int how);
#ifdef KDTRACE_HOOKS
static int owner_lockmgr(struct lock_object *lock, struct thread **owner);
#endif
static int unlock_lockmgr(struct lock_object *lock);
struct lock_class lock_class_lockmgr = {
.lc_name = "lockmgr",
.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
.lc_assert = assert_lockmgr,
#ifdef DDB
.lc_ddb_show = db_show_lockmgr,
#endif
.lc_lock = lock_lockmgr,
.lc_unlock = unlock_lockmgr,
#ifdef KDTRACE_HOOKS
.lc_owner = owner_lockmgr,
#endif
};
#ifdef ADAPTIVE_LOCKMGRS
static u_int alk_retries = 10;
static u_int alk_loops = 10000;
SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging");
SYSCTL_UINT(_debug_lockmgr, OID_AUTO, retries, CTLFLAG_RW, &alk_retries, 0, "");
SYSCTL_UINT(_debug_lockmgr, OID_AUTO, loops, CTLFLAG_RW, &alk_loops, 0, "");
#endif
static __inline struct thread *
lockmgr_xholder(struct lock *lk)
{
uintptr_t x;
x = lk->lk_lock;
return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
}
/*
* It assumes sleepq_lock held and returns with this one unheld.
* It also assumes the generic interlock is sane and previously checked.
* If LK_INTERLOCK is specified the interlock is not reacquired after the
* sleep.
*/
static __inline int
sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *wmesg, int pri, int timo, int queue)
{
GIANT_DECLARE;
struct lock_class *class;
int catch, error;
class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
catch = pri & PCATCH;
pri &= PRIMASK;
error = 0;
LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
(queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
GIANT_SAVE();
sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
SLEEPQ_INTERRUPTIBLE : 0), queue);
if ((flags & LK_TIMELOCK) && timo)
sleepq_set_timeout(&lk->lock_object, timo);
/*
* Decisional switch for real sleeping.
*/
if ((flags & LK_TIMELOCK) && timo && catch)
error = sleepq_timedwait_sig(&lk->lock_object, pri);
else if ((flags & LK_TIMELOCK) && timo)
error = sleepq_timedwait(&lk->lock_object, pri);
else if (catch)
error = sleepq_wait_sig(&lk->lock_object, pri);
else
sleepq_wait(&lk->lock_object, pri);
GIANT_RESTORE();
if ((flags & LK_SLEEPFAIL) && error == 0)
error = ENOLCK;
return (error);
}
static __inline int
wakeupshlk(struct lock *lk, const char *file, int line)
{
uintptr_t v, x;
int queue, wakeup_swapper;
TD_LOCKS_DEC(curthread);
TD_SLOCKS_DEC(curthread);
WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
wakeup_swapper = 0;
for (;;) {
x = lk->lk_lock;
/*
* If there is more than one shared lock held, just drop one
* and return.
*/
if (LK_SHARERS(x) > 1) {
if (atomic_cmpset_ptr(&lk->lk_lock, x,
x - LK_ONE_SHARER))
break;
continue;
}
/*
* If there are not waiters on the exclusive queue, drop the
* lock quickly.
*/
if ((x & LK_ALL_WAITERS) == 0) {
MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
LK_SHARERS_LOCK(1));
if (atomic_cmpset_ptr(&lk->lk_lock, x, LK_UNLOCKED))
break;
continue;
}
/*
* We should have a sharer with waiters, so enter the hard
* path in order to handle wakeups correctly.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
v = LK_UNLOCKED;
/*
* If the lock has exclusive waiters, give them preference in
* order to avoid deadlock with shared runners up.
*/
if (x & LK_EXCLUSIVE_WAITERS) {
queue = SQ_EXCLUSIVE_QUEUE;
v |= (x & LK_SHARED_WAITERS);
} else {
MPASS((x & ~LK_EXCLUSIVE_SPINNERS) ==
LK_SHARED_WAITERS);
queue = SQ_SHARED_QUEUE;
}
if (!atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
v)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
0, queue);
sleepq_release(&lk->lock_object);
break;
}
lock_profile_release_lock(&lk->lock_object);
return (wakeup_swapper);
}
static void
assert_lockmgr(struct lock_object *lock, int what)
{
panic("lockmgr locks do not support assertions");
}
static void
lock_lockmgr(struct lock_object *lock, int how)
{
panic("lockmgr locks do not support sleep interlocking");
}
static int
unlock_lockmgr(struct lock_object *lock)
{
panic("lockmgr locks do not support sleep interlocking");
}
#ifdef KDTRACE_HOOKS
static int
owner_lockmgr(struct lock_object *lock, struct thread **owner)
{
panic("lockmgr locks do not support owner inquiring");
}
#endif
void
lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
{
int iflags;
MPASS((flags & ~LK_INIT_MASK) == 0);
iflags = LO_SLEEPABLE | LO_UPGRADABLE;
if (flags & LK_CANRECURSE)
iflags |= LO_RECURSABLE;
if ((flags & LK_NODUP) == 0)
iflags |= LO_DUPOK;
if (flags & LK_NOPROFILE)
iflags |= LO_NOPROFILE;
if ((flags & LK_NOWITNESS) == 0)
iflags |= LO_WITNESS;
if (flags & LK_QUIET)
iflags |= LO_QUIET;
iflags |= flags & (LK_ADAPTIVE | LK_NOSHARE);
lk->lk_lock = LK_UNLOCKED;
lk->lk_recurse = 0;
lk->lk_timo = timo;
lk->lk_pri = pri;
lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
STACK_ZERO(lk);
}
void
lockdestroy(struct lock *lk)
{
KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
lock_destroy(&lk->lock_object);
}
int
__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *wmesg, int pri, int timo, const char *file, int line)
{
GIANT_DECLARE;
struct lock_class *class;
const char *iwmesg;
uintptr_t tid, v, x;
u_int op;
int error, ipri, itimo, queue, wakeup_swapper;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
#ifdef ADAPTIVE_LOCKMGRS
volatile struct thread *owner;
u_int i, spintries = 0;
#endif
error = 0;
tid = (uintptr_t)curthread;
op = (flags & LK_TYPE_MASK);
iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
MPASS((flags & ~LK_TOTAL_MASK) == 0);
KASSERT((op & (op - 1)) == 0,
("%s: Invalid requested operation @ %s:%d", __func__, file, line));
KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
(op != LK_DOWNGRADE && op != LK_RELEASE),
("%s: Invalid flags in regard of the operation desired @ %s:%d",
__func__, file, line));
KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
__func__, file, line));
class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
if (panicstr != NULL) {
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
return (0);
}
if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
op = LK_EXCLUSIVE;
wakeup_swapper = 0;
switch (op) {
case LK_SHARED:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
file, line, ilk);
for (;;) {
x = lk->lk_lock;
/*
* If no other thread has an exclusive lock, or
* no exclusive waiter is present, bump the count of
* sharers. Since we have to preserve the state of
* waiters, if we fail to acquire the shared lock
* loop back and retry.
*/
if (LK_CAN_SHARE(x)) {
if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
x + LK_ONE_SHARER))
break;
continue;
}
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is already held by curthread in
* exclusive way avoid a deadlock.
*/
if (LK_HOLDER(x) == tid) {
LOCK_LOG2(lk,
"%s: %p already held in exclusive mode",
__func__, lk);
error = EDEADLK;
break;
}
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
#ifdef ADAPTIVE_LOCKMGRS
/*
* If the owner is running on another CPU, spin until
* the owner stops running or the state of the lock
* changes.
*/
if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
LK_HOLDER(x) != LK_KERNPROC) {
owner = (struct thread *)LK_HOLDER(x);
if (LOCK_LOG_TEST(&lk->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
__func__, lk, owner);
/*
* If we are holding also an interlock drop it
* in order to avoid a deadlock if the lockmgr
* owner is adaptively spinning on the
* interlock itself.
*/
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
while (LK_HOLDER(lk->lk_lock) ==
(uintptr_t)owner && TD_IS_RUNNING(owner))
cpu_spinwait();
} else if (LK_CAN_ADAPT(lk, flags) &&
(x & LK_SHARE) !=0 && LK_SHARERS(x) &&
spintries < alk_retries) {
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
spintries++;
for (i = 0; i < alk_loops; i++) {
if (LOCK_LOG_TEST(&lk->lock_object, 0))
CTR4(KTR_LOCK,
"%s: shared spinning on %p with %u and %u",
__func__, lk, spintries, i);
x = lk->lk_lock;
if ((x & LK_SHARE) == 0 ||
LK_CAN_SHARE(x) != 0)
break;
cpu_spinwait();
}
if (i != alk_loops)
continue;
}
#endif
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
/*
* if the lock can be acquired in shared mode, try
* again.
*/
if (LK_CAN_SHARE(x)) {
sleepq_release(&lk->lock_object);
continue;
}
#ifdef ADAPTIVE_LOCKMGRS
/*
* The current lock owner might have started executing
* on another CPU (or the lock could have changed
* owner) while we were waiting on the turnstile
* chain lock. If so, drop the turnstile lock and try
* again.
*/
if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
LK_HOLDER(x) != LK_KERNPROC) {
owner = (struct thread *)LK_HOLDER(x);
if (TD_IS_RUNNING(owner)) {
sleepq_release(&lk->lock_object);
continue;
}
}
#endif
/*
* Try to set the LK_SHARED_WAITERS flag. If we fail,
* loop back and retry.
*/
if ((x & LK_SHARED_WAITERS) == 0) {
if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
x | LK_SHARED_WAITERS)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG2(lk, "%s: %p set shared waiters flag",
__func__, lk);
}
/*
* As far as we have been unable to acquire the
* shared lock and the shared waiters flag is set,
* we will sleep.
*/
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_SHARED_QUEUE);
flags &= ~LK_INTERLOCK;
if (error) {
LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",
__func__, lk, error);
break;
}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
lock_profile_obtain_lock_success(&lk->lock_object,
contested, waittime, file, line);
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
line);
WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file,
line);
TD_LOCKS_INC(curthread);
TD_SLOCKS_INC(curthread);
STACK_SAVE(lk);
}
break;
case LK_UPGRADE:
_lockmgr_assert(lk, KA_SLOCKED, file, line);
v = lk->lk_lock;
x = v & LK_ALL_WAITERS;
v &= LK_EXCLUSIVE_SPINNERS;
/*
* Try to switch from one shared lock to an exclusive one.
* We need to preserve waiters flags during the operation.
*/
if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
tid | x)) {
LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
line);
WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_SLOCKS_DEC(curthread);
break;
}
/*
* We have been unable to succeed in upgrading, so just
* give up the shared lock.
*/
wakeup_swapper |= wakeupshlk(lk, file, line);
/* FALLTHROUGH */
case LK_EXCLUSIVE:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, ilk);
/*
* If curthread already holds the lock and this one is
* allowed to recurse, simply recurse on it.
*/
if (lockmgr_xlocked(lk)) {
if ((flags & LK_CANRECURSE) == 0 &&
(lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
/*
* If the lock is expected to not panic just
* give up and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk,
"%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
__func__, iwmesg, file, line);
}
lk->lk_recurse++;
LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
break;
}
while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
tid)) {
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
#ifdef ADAPTIVE_LOCKMGRS
/*
* If the owner is running on another CPU, spin until
* the owner stops running or the state of the lock
* changes.
*/
x = lk->lk_lock;
if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
LK_HOLDER(x) != LK_KERNPROC) {
owner = (struct thread *)LK_HOLDER(x);
if (LOCK_LOG_TEST(&lk->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
__func__, lk, owner);
/*
* If we are holding also an interlock drop it
* in order to avoid a deadlock if the lockmgr
* owner is adaptively spinning on the
* interlock itself.
*/
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
while (LK_HOLDER(lk->lk_lock) ==
(uintptr_t)owner && TD_IS_RUNNING(owner))
cpu_spinwait();
} else if (LK_CAN_ADAPT(lk, flags) &&
(x & LK_SHARE) != 0 && LK_SHARERS(x) &&
spintries < alk_retries) {
if ((x & LK_EXCLUSIVE_SPINNERS) == 0 &&
!atomic_cmpset_ptr(&lk->lk_lock, x,
x | LK_EXCLUSIVE_SPINNERS))
continue;
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
spintries++;
for (i = 0; i < alk_loops; i++) {
if (LOCK_LOG_TEST(&lk->lock_object, 0))
CTR4(KTR_LOCK,
"%s: shared spinning on %p with %u and %u",
__func__, lk, spintries, i);
if ((lk->lk_lock &
LK_EXCLUSIVE_SPINNERS) == 0)
break;
cpu_spinwait();
}
if (i != alk_loops)
continue;
}
#endif
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
/*
* if the lock has been released while we spun on
* the sleepqueue chain lock just try again.
*/
if (x == LK_UNLOCKED) {
sleepq_release(&lk->lock_object);
continue;
}
#ifdef ADAPTIVE_LOCKMGRS
/*
* The current lock owner might have started executing
* on another CPU (or the lock could have changed
* owner) while we were waiting on the turnstile
* chain lock. If so, drop the turnstile lock and try
* again.
*/
if (LK_CAN_ADAPT(lk, flags) && (x & LK_SHARE) == 0 &&
LK_HOLDER(x) != LK_KERNPROC) {
owner = (struct thread *)LK_HOLDER(x);
if (TD_IS_RUNNING(owner)) {
sleepq_release(&lk->lock_object);
continue;
}
}
#endif
/*
* The lock can be in the state where there is a
* pending queue of waiters, but still no owner.
* This happens when the lock is contested and an
* owner is going to claim the lock.
* If curthread is the one successfully acquiring it
* claim lock ownership and return, preserving waiters
* flags.
*/
v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
if ((x & ~v) == LK_UNLOCKED) {
v &= ~LK_EXCLUSIVE_SPINNERS;
if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
tid | v)) {
sleepq_release(&lk->lock_object);
LOCK_LOG2(lk,
"%s: %p claimed by a new writer",
__func__, lk);
break;
}
sleepq_release(&lk->lock_object);
continue;
}
/*
* Try to set the LK_EXCLUSIVE_WAITERS flag. If we
* fail, loop back and retry.
*/
if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
if (!atomic_cmpset_ptr(&lk->lk_lock, x,
x | LK_EXCLUSIVE_WAITERS)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG2(lk, "%s: %p set excl waiters flag",
__func__, lk);
}
/*
* As far as we have been unable to acquire the
* exclusive lock and the exclusive waiters flag
* is set, we will sleep.
*/
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_EXCLUSIVE_QUEUE);
flags &= ~LK_INTERLOCK;
if (error) {
LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",
__func__, lk, error);
break;
}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
lock_profile_obtain_lock_success(&lk->lock_object,
contested, waittime, file, line);
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
STACK_SAVE(lk);
}
break;
case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
TD_SLOCKS_INC(curthread);
/*
* In order to preserve waiters flags, just spin.
*/
for (;;) {
x = lk->lk_lock;
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
x &= LK_ALL_WAITERS;
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
LK_SHARERS_LOCK(1) | x))
break;
cpu_spinwait();
}
break;
case LK_RELEASE:
_lockmgr_assert(lk, KA_LOCKED, file, line);
x = lk->lk_lock;
if ((x & LK_SHARE) == 0) {
/*
* As first option, treact the lock as if it has not
* any waiter.
* Fix-up the tid var if the lock has been disowned.
*/
if (LK_HOLDER(x) == LK_KERNPROC)
tid = LK_KERNPROC;
else {
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE,
file, line);
TD_LOCKS_DEC(curthread);
}
LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
lk->lk_recurse, file, line);
/*
* The lock is held in exclusive mode.
* If the lock is recursed also, then unrecurse it.
*/
if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
lk);
lk->lk_recurse--;
break;
}
if (tid != LK_KERNPROC)
lock_profile_release_lock(&lk->lock_object);
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
LK_UNLOCKED))
break;
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
v = LK_UNLOCKED;
/*
* If the lock has exclusive waiters, give them
* preference in order to avoid deadlock with
* shared runners up.
*/
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
if (x & LK_EXCLUSIVE_WAITERS) {
queue = SQ_EXCLUSIVE_QUEUE;
v |= (x & LK_SHARED_WAITERS);
} else {
MPASS((x & LK_ALL_WAITERS) ==
LK_SHARED_WAITERS);
queue = SQ_SHARED_QUEUE;
}
LOCK_LOG3(lk,
"%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
atomic_store_rel_ptr(&lk->lk_lock, v);
wakeup_swapper = sleepq_broadcast(&lk->lock_object,
SLEEPQ_LK, 0, queue);
sleepq_release(&lk->lock_object);
break;
} else
wakeup_swapper = wakeupshlk(lk, file, line);
break;
case LK_DRAIN:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, ilk);
/*
* Trying to drain a lock we already own will result in a
* deadlock.
*/
if (lockmgr_xlocked(lk)) {
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: draining %s with the lock held @ %s:%d\n",
__func__, iwmesg, file, line);
}
while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
/*
* if the lock has been released while we spun on
* the sleepqueue chain lock just try again.
*/
if (x == LK_UNLOCKED) {
sleepq_release(&lk->lock_object);
continue;
}
v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
if ((x & ~v) == LK_UNLOCKED) {
v = (x & ~LK_EXCLUSIVE_SPINNERS);
if (v & LK_EXCLUSIVE_WAITERS) {
queue = SQ_EXCLUSIVE_QUEUE;
v &= ~LK_EXCLUSIVE_WAITERS;
} else {
MPASS(v & LK_SHARED_WAITERS);
queue = SQ_SHARED_QUEUE;
v &= ~LK_SHARED_WAITERS;
}
if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG3(lk,
"%s: %p waking up all threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ?
"shared" : "exclusive");
wakeup_swapper |= sleepq_broadcast(
&lk->lock_object, SLEEPQ_LK, 0, queue);
/*
* If shared waiters have been woken up we need
* to wait for one of them to acquire the lock
* before to set the exclusive waiters in
* order to avoid a deadlock.
*/
if (queue == SQ_SHARED_QUEUE) {
for (v = lk->lk_lock;
(v & LK_SHARE) && !LK_SHARERS(v);
v = lk->lk_lock)
cpu_spinwait();
}
}
/*
* Try to set the LK_EXCLUSIVE_WAITERS flag. If we
* fail, loop back and retry.
*/
if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
if (!atomic_cmpset_ptr(&lk->lk_lock, x,
x | LK_EXCLUSIVE_WAITERS)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG2(lk, "%s: %p set drain waiters flag",
__func__, lk);
}
/*
* As far as we have been unable to acquire the
* exclusive lock and the exclusive waiters flag
* is set, we will sleep.
*/
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
SQ_EXCLUSIVE_QUEUE);
sleepq_wait(&lk->lock_object, ipri & PRIMASK);
GIANT_RESTORE();
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
lock_profile_obtain_lock_success(&lk->lock_object,
contested, waittime, file, line);
LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
STACK_SAVE(lk);
}
break;
default:
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
}
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
if (wakeup_swapper)
kick_proc0();
return (error);
}
void
_lockmgr_disown(struct lock *lk, const char *file, int line)
{
uintptr_t tid, x;
tid = (uintptr_t)curthread;
_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
/*
* If the owner is already LK_KERNPROC just skip the whole operation.
*/
if (LK_HOLDER(lk->lk_lock) != tid)
return;
lock_profile_release_lock(&lk->lock_object);
LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);
/*
* In order to preserve waiters flags, just spin.
*/
for (;;) {
x = lk->lk_lock;
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
x &= LK_ALL_WAITERS;
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
LK_KERNPROC | x))
return;
cpu_spinwait();
}
}
void
lockmgr_printinfo(struct lock *lk)
{
struct thread *td;
uintptr_t x;
if (lk->lk_lock == LK_UNLOCKED)
printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
else if (lk->lk_lock & LK_SHARE)
printf("lock type %s: SHARED (count %ju)\n",
lk->lock_object.lo_name,
(uintmax_t)LK_SHARERS(lk->lk_lock));
else {
td = lockmgr_xholder(lk);
printf("lock type %s: EXCL by thread %p (pid %d)\n",
lk->lock_object.lo_name, td, td->td_proc->p_pid);
}
x = lk->lk_lock;
if (x & LK_EXCLUSIVE_WAITERS)
printf(" with exclusive waiters pending\n");
if (x & LK_SHARED_WAITERS)
printf(" with shared waiters pending\n");
if (x & LK_EXCLUSIVE_SPINNERS)
printf(" with exclusive spinners pending\n");
STACK_PRINT(lk);
}
int
lockstatus(struct lock *lk)
{
uintptr_t v, x;
int ret;
ret = LK_SHARED;
x = lk->lk_lock;
v = LK_HOLDER(x);
if ((x & LK_SHARE) == 0) {
if (v == (uintptr_t)curthread || v == LK_KERNPROC)
ret = LK_EXCLUSIVE;
else
ret = LK_EXCLOTHER;
} else if (x == LK_UNLOCKED)
ret = 0;
return (ret);
}
#ifdef INVARIANT_SUPPORT
#ifndef INVARIANTS
#undef _lockmgr_assert
#endif
void
_lockmgr_assert(struct lock *lk, int what, const char *file, int line)
{
int slocked = 0;
if (panicstr != NULL)
return;
switch (what) {
case KA_SLOCKED:
case KA_SLOCKED | KA_NOTRECURSED:
case KA_SLOCKED | KA_RECURSED:
slocked = 1;
case KA_LOCKED:
case KA_LOCKED | KA_NOTRECURSED:
case KA_LOCKED | KA_RECURSED:
#ifdef WITNESS
/*
* We cannot trust WITNESS if the lock is held in exclusive
* mode and a call to lockmgr_disown() happened.
* Workaround this skipping the check if the lock is held in
* exclusive mode even for the KA_LOCKED case.
*/
if (slocked || (lk->lk_lock & LK_SHARE)) {
witness_assert(&lk->lock_object, what, file, line);
break;
}
#endif
if (lk->lk_lock == LK_UNLOCKED ||
((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
(!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
panic("Lock %s not %slocked @ %s:%d\n",
lk->lock_object.lo_name, slocked ? "share" : "",
file, line);
if ((lk->lk_lock & LK_SHARE) == 0) {
if (lockmgr_recursed(lk)) {
if (what & KA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
lk->lock_object.lo_name, file,
line);
} else if (what & KA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
}
break;
case KA_XLOCKED:
case KA_XLOCKED | KA_NOTRECURSED:
case KA_XLOCKED | KA_RECURSED:
if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
panic("Lock %s not exclusively locked @ %s:%d\n",
lk->lock_object.lo_name, file, line);
if (lockmgr_recursed(lk)) {
if (what & KA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
} else if (what & KA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
break;
case KA_UNLOCKED:
if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
panic("Lock %s exclusively locked @ %s:%d\n",
lk->lock_object.lo_name, file, line);
break;
default:
panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
line);
}
}
#endif
#ifdef DDB
int
lockmgr_chain(struct thread *td, struct thread **ownerp)
{
struct lock *lk;
lk = td->td_wchan;
if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
return (0);
db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
if (lk->lk_lock & LK_SHARE)
db_printf("SHARED (count %ju)\n",
(uintmax_t)LK_SHARERS(lk->lk_lock));
else
db_printf("EXCL\n");
*ownerp = lockmgr_xholder(lk);
return (1);
}
static void
db_show_lockmgr(struct lock_object *lock)
{
struct thread *td;
struct lock *lk;
lk = (struct lock *)lock;
db_printf(" state: ");
if (lk->lk_lock == LK_UNLOCKED)
db_printf("UNLOCKED\n");
else if (lk->lk_lock & LK_SHARE)
db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
else {
td = lockmgr_xholder(lk);
if (td == (struct thread *)LK_KERNPROC)
db_printf("XLOCK: LK_KERNPROC\n");
else
db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid,
td->td_proc->p_comm);
if (lockmgr_recursed(lk))
db_printf(" recursed: %d\n", lk->lk_recurse);
}
db_printf(" waiters: ");
switch (lk->lk_lock & LK_ALL_WAITERS) {
case LK_SHARED_WAITERS:
db_printf("shared\n");
break;
case LK_EXCLUSIVE_WAITERS:
db_printf("exclusive\n");
break;
case LK_ALL_WAITERS:
db_printf("shared and exclusive\n");
break;
default:
db_printf("none\n");
}
db_printf(" spinners: ");
if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
db_printf("exclusive\n");
else
db_printf("none\n");
}
#endif
Reference in New Issue View Git Blame Copy Permalink