talexander
/
freebsd_amp_hwpstate
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove MUTEX_DECLARE() and MTX_COLD. Instead, postpone full mutex 

initialization until after malloc() is safe to call, then iterate through
all mutexes and complete their initialization.

This change is necessary in order to avoid some circular bootstrapping
dependencies.
This commit is contained in:
Jason Evans 2001-01-21 07:52:20 +00:00
parent 8363b95b57
commit d1c1b8413e
24 changed files with 301 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
share/man/man9/Makefile
View File

@ -148,7 +148,6 @@ MLINKS+=microtime.9 getnanotime.9
MLINKS+=microuptime.9 getmicrouptime.9 microuptime.9 nanouptime.9
MLINKS+=microuptime.9 getnanouptime.9
MLINKS+=mutex.9 MUTEX_DECLARE.9
MLINKS+=mutex.9 mtx_init.9
MLINKS+=mutex.9 mtx_enter.9
MLINKS+=mutex.9 mtx_try_enter.9

40
share/man/man9/mutex.9
View File

@ -39,8 +39,7 @@
.Nm mtx_exit ,
.Nm mtx_destroy ,
.Nm mtx_owned ,
.Nm mtx_assert ,
.Nm MUTEX_DECLARE
.Nm mtx_assert
.Nd kernel synchronization primitives
.Sh SYNOPSIS
.Fd #include <sys/mutex.h>
@ -58,7 +57,6 @@
.Fn mtx_owned "struct mtx *mutex"
.Ft void
.Fn mtx_assert "struct mtx *mutex" "int what"
.Fn MUTEX_DECLARE "modifiers" "name"
.Sh DESCRIPTION
Mutexes are the most basic and primary method of process synchronization.
The major design considerations for mutexes are:
@ -226,34 +224,6 @@ This assertion is only valid in conjuction with
.Dv MA_OWNED .
.El
.Pp
The
.Fn MUTEX_DECLARE
macro is used to declare a mutex that is initialized before
.Xr malloc 9
is operating.
Unfortunately, mutex initialization may require
.Xr malloc 9 .
However, some mutexes are initialized and used before
.Xr malloc 9
can be used.
Declaring these mutexes with the
.Fn MUTEX_DECLARE
macro and then using the
.Dv MTX_COLD
flag when calling
.Fn mtx_init
allows these early mutexes to be initialized and used before
.Xr malloc 9
is available.
The
.Ar modifiers
argument is a list of attributes to be applied to the mutex structure being
declared such as
.Dq static .
The
.Ar name
argument is the name of the mutex structure to declare.
.Pp
The type of a mutex is not an attribute of the mutex,
but instead a function of the
.Fa flags
@ -398,14 +368,6 @@ This should be specified when it is known
that the lock will usually remain unavailable for some time
when it is not immediately available
(i.e.: coarse grained locks protecting large subsystems).
.It Dv MTX_COLD
This option is only used in
.Fn mtx_init
and is used in conjunction with mutexes declared with
.Fn MUTEX_DECLARE
to initialize mutexes that are needed before
.Xr malloc 9
is available for use.
.It Dv MTX_QUIET
This option is used to quiet logging messages during mutex operations.
This can be used to trim superfluous logging messages for debugging purposes.

8
sys/alpha/alpha/machdep.c
View File

@ -154,8 +154,8 @@ struct bootinfo_kernel bootinfo;
struct cpuhead cpuhead;
MUTEX_DECLARE( ,sched_lock);
MUTEX_DECLARE( ,Giant);
struct mtx sched_lock;
struct mtx Giant;
struct user *proc0paddr;
@ -1003,8 +1003,8 @@ alpha_init(pfn, ptb, bim, bip, biv)
/*
* Initialise mutexes.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
/*
* Look at arguments passed to us and compute boothowto.

10
sys/amd64/amd64/machdep.c
View File

@ -249,8 +249,8 @@ static struct globaldata __globaldata;
struct cpuhead cpuhead;
MUTEX_DECLARE(,sched_lock);
MUTEX_DECLARE(,Giant);
struct mtx sched_lock;
struct mtx Giant;
static void
cpu_startup(dummy)
@ -441,7 +441,7 @@ again:
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
#ifdef SMP
/*
@ -1939,7 +1939,7 @@ init386(first)
/*
* We need this mutex before the console probe.
*/
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
/*
* Initialize the console before we print anything out.
@ -1954,7 +1954,7 @@ init386(first)
/*
* Giant is used early for at least debugger traps and unexpected traps.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
#ifdef DDB
kdb_init();

2
sys/amd64/amd64/tsc.c
View File

@ -140,7 +140,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

2
sys/amd64/isa/clock.c
View File

@ -140,7 +140,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

10
sys/i386/i386/machdep.c
View File

@ -249,8 +249,8 @@ static struct globaldata __globaldata;
struct cpuhead cpuhead;
MUTEX_DECLARE(,sched_lock);
MUTEX_DECLARE(,Giant);
struct mtx sched_lock;
struct mtx Giant;
static void
cpu_startup(dummy)
@ -441,7 +441,7 @@ again:
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
#ifdef SMP
/*
@ -1939,7 +1939,7 @@ init386(first)
/*
* We need this mutex before the console probe.
*/
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
/*
* Initialize the console before we print anything out.
@ -1954,7 +1954,7 @@ init386(first)
/*
* Giant is used early for at least debugger traps and unexpected traps.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
#ifdef DDB
kdb_init();

2
sys/i386/i386/tsc.c
View File

@ -140,7 +140,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

4
sys/i386/i386/vm86.c
View File

@ -50,7 +50,7 @@ extern int i386_extend_pcb __P((struct proc *));
extern int vm86pa;
extern struct pcb *vm86pcb;
MUTEX_DECLARE(static, vm86pcb_lock);
static struct mtx vm86pcb_lock;
extern int vm86_bioscall(struct vm86frame *);
extern void vm86_biosret(struct vm86frame *);
@ -426,7 +426,7 @@ vm86_initialize(void)
pcb = &vml->vml_pcb;
ext = &vml->vml_ext;
mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF | MTX_COLD);
mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF);
bzero(pcb, sizeof(struct pcb));
pcb->new_ptd = vm86pa | PG_V | PG_RW | PG_U;

2
sys/i386/isa/clock.c
View File

@ -140,7 +140,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

8
sys/ia64/ia64/machdep.c
View File

@ -84,8 +84,8 @@ struct bootinfo_kernel bootinfo;
struct cpuhead cpuhead;
MUTEX_DECLARE( ,sched_lock);
MUTEX_DECLARE( ,Giant);
struct mtx sched_lock;
struct mtx Giant;
struct user *proc0paddr;
@ -590,8 +590,8 @@ ia64_init()
/*
* Initialise mutexes.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
#if 0
/*

2
sys/isa/atrtc.c
View File

@ -140,7 +140,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

6
sys/kern/kern_lock.c
View File

@ -80,7 +80,7 @@
extern int lock_nmtx;
int lock_mtx_selector;
struct mtx *lock_mtx_array;
MUTEX_DECLARE(static, lock_mtx);
static struct mtx lock_mtx;
static int acquire(struct lock *lkp, int extflags, int wanted);
static int apause(struct lock *lkp, int flags);
@ -98,7 +98,7 @@ lockmgr_init(void *dummy __unused)
* initialized in a call to lockinit().
*/
if (lock_mtx_selector == 0)
mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD);
mtx_init(&lock_mtx, "lockmgr", MTX_DEF);
else {
/*
* This is necessary if (lock_nmtx == 1) and doesn't hurt
@ -546,7 +546,7 @@ lockinit(lkp, prio, wmesg, timo, flags)
* so there's no reason to protect modification of
* lock_mtx_selector or lock_mtx.
*/
mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD);
mtx_init(&lock_mtx, "lockmgr", MTX_DEF);
lock_mtx_selector = 1;
}
lkp->lk_interlock = &lock_mtx;

4
sys/kern/kern_malloc.c
View File

@ -74,7 +74,7 @@ static struct kmemusage *kmemusage;
static char *kmembase;
static char *kmemlimit;
MUTEX_DECLARE(static, malloc_mtx);
static struct mtx malloc_mtx;
u_int vm_kmem_size;
@ -439,7 +439,7 @@ kmeminit(dummy)
#error "kmeminit: MAXALLOCSAVE too small"
#endif
mtx_init(&malloc_mtx, "malloc", MTX_DEF | MTX_COLD);
mtx_init(&malloc_mtx, "malloc", MTX_DEF);
/*
* Try to auto-tune the kernel memory size, so that it is

110
sys/kern/kern_mutex.c
View File

@ -88,11 +88,16 @@
#ifdef WITNESS
static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
"All mutexes queue head" };
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug,
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
/*
* Set to 0 once mutexes have been fully initialized so that witness code can be
* safely executed.
*/
static int witness_cold = 1;
#else /* WITNESS */
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head",
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
#endif /* WITNESS */
@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when)
int i;
int retval = 0;
#ifdef WITNESS
if (witness_cold)
return 0;
#endif
if (m == &all_mtx || cold)
return 0;
@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when)
void
mtx_init(struct mtx *m, const char *t, int flag)
{
#ifdef WITNESS
struct mtx_debug *debug;
#endif
if ((flag & MTX_QUIET) == 0)
CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t);
#ifdef MUTEX_DEBUG
if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */
return;
#endif
#ifdef WITNESS
if (flag & MTX_COLD)
debug = m->mtx_debug;
else
debug = NULL;
if (debug == NULL) {
#ifdef DIAGNOSTIC
if(cold && bootverbose)
printf("malloc'ing mtx_debug while cold for %s\n", t);
#endif
/* XXX - should not use DEVBUF */
debug = malloc(sizeof(struct mtx_debug), M_DEVBUF,
M_NOWAIT | M_ZERO);
MPASS(debug != NULL);
}
#endif
bzero((void *)m, sizeof *m);
TAILQ_INIT(&m->mtx_blocked);
#ifdef WITNESS
m->mtx_debug = debug;
#endif
if (!witness_cold) {
/* XXX - should not use DEVBUF */
m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(m->mtx_union.mtxu_debug != NULL);
m->mtx_description = t;
} else {
/*
* Save a pointer to the description so that witness_fixup()
* can properly initialize this mutex later on.
*/
m->mtx_union.mtxu_description = t;
}
#else
m->mtx_description = t;
#endif
m->mtx_flags = flag;
m->mtx_lock = MTX_UNOWNED;
/* Put on all mutex queue */
mtx_enter(&all_mtx, MTX_DEF);
@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag)
if (++mtx_cur_cnt > mtx_max_cnt)
mtx_max_cnt = mtx_cur_cnt;
mtx_exit(&all_mtx, MTX_DEF);
witness_init(m, flag);
#ifdef WITNESS
if (!witness_cold)
witness_init(m, flag);
#endif
}
void
mtx_destroy(struct mtx *m)
{
#ifdef WITNESS
KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
__FUNCTION__));
#endif
CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description);
#ifdef MUTEX_DEBUG
if (m->mtx_next == NULL)
@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m)
m->mtx_next = m->mtx_prev = NULL;
#endif
#ifdef WITNESS
free(m->mtx_debug, M_DEVBUF);
m->mtx_debug = NULL;
free(m->mtx_union.mtxu_debug, M_DEVBUF);
m->mtx_union.mtxu_debug = NULL;
#endif
mtx_cur_cnt--;
mtx_exit(&all_mtx, MTX_DEF);
}
static void
witness_fixup(void *dummy __unused)
{
#ifdef WITNESS
struct mtx *mp;
const char *description;
/* Iterate through all mutexes and finish up mutex initialization. */
for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
description = mp->mtx_union.mtxu_description;
/* XXX - should not use DEVBUF */
mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(mp->mtx_union.mtxu_debug != NULL);
mp->mtx_description = description;
witness_init(mp, mp->mtx_flags);
}
/* Mark the witness code as being ready for use. */
atomic_store_rel_int(&witness_cold, 0);
#endif
}
SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
/*
* The non-inlined versions of the mtx_*() functions are always built (above),
* but the witness code depends on the WITNESS kernel option being specified.
@ -716,7 +755,7 @@ int witness_skipspin = 0;
SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
"");
MUTEX_DECLARE(static,w_mtx);
static struct mtx w_mtx;
static struct witness *w_free;
static struct witness *w_all;
static int w_inited;
@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
int go_into_ddb = 0;
#endif /* DDB */
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
{
struct witness *w;
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
struct proc *p;
struct witness *w = m->mtx_witness;
if (witness_cold)
return;
if (panicstr)
return;
if (flags & MTX_SPIN) {
@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...))
{
struct witness *w, *w1;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
witness_levelall();
for (w = w_all; w; w = w->w_next) {
@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
char **sleep;
int n = 0;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
p = CURPROC;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1122,7 +1169,7 @@ enroll(const char *description, int flag)
return (NULL);
if (w_inited == 0) {
mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN);
mtx_init(&w_mtx, "witness lock", MTX_SPIN);
for (i = 0; i < WITNESS_COUNT; i++) {
w = &w_data[i];
witness_free(w);
@ -1401,6 +1448,7 @@ witness_list(struct proc *p)
struct mtx *m;
int nheld;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
nheld = 0;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1416,6 +1464,8 @@ witness_list(struct proc *p)
void
witness_save(struct mtx *m, const char **filep, int *linep)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
*filep = m->mtx_witness->w_file;
*linep = m->mtx_witness->w_line;
}
@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep)
void
witness_restore(struct mtx *m, const char *file, int line)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
m->mtx_witness->w_file = file;
m->mtx_witness->w_line = line;
}

110
sys/kern/subr_turnstile.c
View File

@ -88,11 +88,16 @@
#ifdef WITNESS
static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
"All mutexes queue head" };
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug,
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
/*
* Set to 0 once mutexes have been fully initialized so that witness code can be
* safely executed.
*/
static int witness_cold = 1;
#else /* WITNESS */
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head",
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
#endif /* WITNESS */
@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when)
int i;
int retval = 0;
#ifdef WITNESS
if (witness_cold)
return 0;
#endif
if (m == &all_mtx || cold)
return 0;
@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when)
void
mtx_init(struct mtx *m, const char *t, int flag)
{
#ifdef WITNESS
struct mtx_debug *debug;
#endif
if ((flag & MTX_QUIET) == 0)
CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t);
#ifdef MUTEX_DEBUG
if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */
return;
#endif
#ifdef WITNESS
if (flag & MTX_COLD)
debug = m->mtx_debug;
else
debug = NULL;
if (debug == NULL) {
#ifdef DIAGNOSTIC
if(cold && bootverbose)
printf("malloc'ing mtx_debug while cold for %s\n", t);
#endif
/* XXX - should not use DEVBUF */
debug = malloc(sizeof(struct mtx_debug), M_DEVBUF,
M_NOWAIT | M_ZERO);
MPASS(debug != NULL);
}
#endif
bzero((void *)m, sizeof *m);
TAILQ_INIT(&m->mtx_blocked);
#ifdef WITNESS
m->mtx_debug = debug;
#endif
if (!witness_cold) {
/* XXX - should not use DEVBUF */
m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(m->mtx_union.mtxu_debug != NULL);
m->mtx_description = t;
} else {
/*
* Save a pointer to the description so that witness_fixup()
* can properly initialize this mutex later on.
*/
m->mtx_union.mtxu_description = t;
}
#else
m->mtx_description = t;
#endif
m->mtx_flags = flag;
m->mtx_lock = MTX_UNOWNED;
/* Put on all mutex queue */
mtx_enter(&all_mtx, MTX_DEF);
@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag)
if (++mtx_cur_cnt > mtx_max_cnt)
mtx_max_cnt = mtx_cur_cnt;
mtx_exit(&all_mtx, MTX_DEF);
witness_init(m, flag);
#ifdef WITNESS
if (!witness_cold)
witness_init(m, flag);
#endif
}
void
mtx_destroy(struct mtx *m)
{
#ifdef WITNESS
KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
__FUNCTION__));
#endif
CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description);
#ifdef MUTEX_DEBUG
if (m->mtx_next == NULL)
@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m)
m->mtx_next = m->mtx_prev = NULL;
#endif
#ifdef WITNESS
free(m->mtx_debug, M_DEVBUF);
m->mtx_debug = NULL;
free(m->mtx_union.mtxu_debug, M_DEVBUF);
m->mtx_union.mtxu_debug = NULL;
#endif
mtx_cur_cnt--;
mtx_exit(&all_mtx, MTX_DEF);
}
static void
witness_fixup(void *dummy __unused)
{
#ifdef WITNESS
struct mtx *mp;
const char *description;
/* Iterate through all mutexes and finish up mutex initialization. */
for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
description = mp->mtx_union.mtxu_description;
/* XXX - should not use DEVBUF */
mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(mp->mtx_union.mtxu_debug != NULL);
mp->mtx_description = description;
witness_init(mp, mp->mtx_flags);
}
/* Mark the witness code as being ready for use. */
atomic_store_rel_int(&witness_cold, 0);
#endif
}
SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
/*
* The non-inlined versions of the mtx_*() functions are always built (above),
* but the witness code depends on the WITNESS kernel option being specified.
@ -716,7 +755,7 @@ int witness_skipspin = 0;
SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
"");
MUTEX_DECLARE(static,w_mtx);
static struct mtx w_mtx;
static struct witness *w_free;
static struct witness *w_all;
static int w_inited;
@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
int go_into_ddb = 0;
#endif /* DDB */
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
{
struct witness *w;
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
struct proc *p;
struct witness *w = m->mtx_witness;
if (witness_cold)
return;
if (panicstr)
return;
if (flags & MTX_SPIN) {
@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...))
{
struct witness *w, *w1;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
witness_levelall();
for (w = w_all; w; w = w->w_next) {
@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
char **sleep;
int n = 0;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
p = CURPROC;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1122,7 +1169,7 @@ enroll(const char *description, int flag)
return (NULL);
if (w_inited == 0) {
mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN);
mtx_init(&w_mtx, "witness lock", MTX_SPIN);
for (i = 0; i < WITNESS_COUNT; i++) {
w = &w_data[i];
witness_free(w);
@ -1401,6 +1448,7 @@ witness_list(struct proc *p)
struct mtx *m;
int nheld;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
nheld = 0;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1416,6 +1464,8 @@ witness_list(struct proc *p)
void
witness_save(struct mtx *m, const char **filep, int *linep)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
*filep = m->mtx_witness->w_file;
*linep = m->mtx_witness->w_line;
}
@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep)
void
witness_restore(struct mtx *m, const char *file, int line)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
m->mtx_witness->w_file = file;
m->mtx_witness->w_line = line;
}

110
sys/kern/subr_witness.c
View File

@ -88,11 +88,16 @@
#ifdef WITNESS
static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
"All mutexes queue head" };
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug,
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
/*
* Set to 0 once mutexes have been fully initialized so that witness code can be
* safely executed.
*/
static int witness_cold = 1;
#else /* WITNESS */
static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head",
static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"},
TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
{ NULL, NULL }, &all_mtx, &all_mtx };
#endif /* WITNESS */
@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when)
int i;
int retval = 0;
#ifdef WITNESS
if (witness_cold)
return 0;
#endif
if (m == &all_mtx || cold)
return 0;
@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when)
void
mtx_init(struct mtx *m, const char *t, int flag)
{
#ifdef WITNESS
struct mtx_debug *debug;
#endif
if ((flag & MTX_QUIET) == 0)
CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t);
#ifdef MUTEX_DEBUG
if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */
return;
#endif
#ifdef WITNESS
if (flag & MTX_COLD)
debug = m->mtx_debug;
else
debug = NULL;
if (debug == NULL) {
#ifdef DIAGNOSTIC
if(cold && bootverbose)
printf("malloc'ing mtx_debug while cold for %s\n", t);
#endif
/* XXX - should not use DEVBUF */
debug = malloc(sizeof(struct mtx_debug), M_DEVBUF,
M_NOWAIT | M_ZERO);
MPASS(debug != NULL);
}
#endif
bzero((void *)m, sizeof *m);
TAILQ_INIT(&m->mtx_blocked);
#ifdef WITNESS
m->mtx_debug = debug;
#endif
if (!witness_cold) {
/* XXX - should not use DEVBUF */
m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(m->mtx_union.mtxu_debug != NULL);
m->mtx_description = t;
} else {
/*
* Save a pointer to the description so that witness_fixup()
* can properly initialize this mutex later on.
*/
m->mtx_union.mtxu_description = t;
}
#else
m->mtx_description = t;
#endif
m->mtx_flags = flag;
m->mtx_lock = MTX_UNOWNED;
/* Put on all mutex queue */
mtx_enter(&all_mtx, MTX_DEF);
@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag)
if (++mtx_cur_cnt > mtx_max_cnt)
mtx_max_cnt = mtx_cur_cnt;
mtx_exit(&all_mtx, MTX_DEF);
witness_init(m, flag);
#ifdef WITNESS
if (!witness_cold)
witness_init(m, flag);
#endif
}
void
mtx_destroy(struct mtx *m)
{
#ifdef WITNESS
KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
__FUNCTION__));
#endif
CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description);
#ifdef MUTEX_DEBUG
if (m->mtx_next == NULL)
@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m)
m->mtx_next = m->mtx_prev = NULL;
#endif
#ifdef WITNESS
free(m->mtx_debug, M_DEVBUF);
m->mtx_debug = NULL;
free(m->mtx_union.mtxu_debug, M_DEVBUF);
m->mtx_union.mtxu_debug = NULL;
#endif
mtx_cur_cnt--;
mtx_exit(&all_mtx, MTX_DEF);
}
static void
witness_fixup(void *dummy __unused)
{
#ifdef WITNESS
struct mtx *mp;
const char *description;
/* Iterate through all mutexes and finish up mutex initialization. */
for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
description = mp->mtx_union.mtxu_description;
/* XXX - should not use DEVBUF */
mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug),
M_DEVBUF, M_NOWAIT | M_ZERO);
MPASS(mp->mtx_union.mtxu_debug != NULL);
mp->mtx_description = description;
witness_init(mp, mp->mtx_flags);
}
/* Mark the witness code as being ready for use. */
atomic_store_rel_int(&witness_cold, 0);
#endif
}
SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
/*
* The non-inlined versions of the mtx_*() functions are always built (above),
* but the witness code depends on the WITNESS kernel option being specified.
@ -716,7 +755,7 @@ int witness_skipspin = 0;
SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
"");
MUTEX_DECLARE(static,w_mtx);
static struct mtx w_mtx;
static struct witness *w_free;
static struct witness *w_all;
static int w_inited;
@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
int go_into_ddb = 0;
#endif /* DDB */
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
{
struct witness *w;
if (witness_cold)
return;
if (panicstr)
return;
w = m->mtx_witness;
@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
struct proc *p;
struct witness *w = m->mtx_witness;
if (witness_cold)
return;
if (panicstr)
return;
if (flags & MTX_SPIN) {
@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...))
{
struct witness *w, *w1;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
witness_levelall();
for (w = w_all; w; w = w->w_next) {
@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
char **sleep;
int n = 0;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
p = CURPROC;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1122,7 +1169,7 @@ enroll(const char *description, int flag)
return (NULL);
if (w_inited == 0) {
mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN);
mtx_init(&w_mtx, "witness lock", MTX_SPIN);
for (i = 0; i < WITNESS_COUNT; i++) {
w = &w_data[i];
witness_free(w);
@ -1401,6 +1448,7 @@ witness_list(struct proc *p)
struct mtx *m;
int nheld;
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
nheld = 0;
for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL;
m = LIST_NEXT(m, mtx_held)) {
@ -1416,6 +1464,8 @@ witness_list(struct proc *p)
void
witness_save(struct mtx *m, const char **filep, int *linep)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
*filep = m->mtx_witness->w_file;
*linep = m->mtx_witness->w_line;
}
@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep)
void
witness_restore(struct mtx *m, const char *file, int line)
{
KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
m->mtx_witness->w_file = file;
m->mtx_witness->w_line = line;
}

2
sys/pc98/cbus/clock.c
View File

@ -154,7 +154,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

2
sys/pc98/cbus/pcrtc.c
View File

@ -154,7 +154,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

10
sys/pc98/i386/machdep.c
View File

@ -262,8 +262,8 @@ static struct globaldata __globaldata;
struct cpuhead cpuhead;
MUTEX_DECLARE(,sched_lock);
MUTEX_DECLARE(,Giant);
struct mtx sched_lock;
struct mtx Giant;
static void
cpu_startup(dummy)
@ -454,7 +454,7 @@ again:
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
#ifdef SMP
/*
@ -2248,7 +2248,7 @@ init386(first)
/*
* We need this mutex before the console probe.
*/
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
/*
* Initialize the console before we print anything out.
@ -2263,7 +2263,7 @@ init386(first)
/*
* Giant is used early for at least debugger traps and unexpected traps.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
#ifdef DDB
kdb_init();

2
sys/pc98/pc98/clock.c
View File

@ -154,7 +154,7 @@ int timer0_max_count;
u_int tsc_freq;
int tsc_is_broken;
int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */
MUTEX_DECLARE(,clock_lock);
struct mtx clock_lock;
static int beeping = 0;
static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};

10
sys/pc98/pc98/machdep.c
View File

@ -262,8 +262,8 @@ static struct globaldata __globaldata;
struct cpuhead cpuhead;
MUTEX_DECLARE(,sched_lock);
MUTEX_DECLARE(,Giant);
struct mtx sched_lock;
struct mtx Giant;
static void
cpu_startup(dummy)
@ -454,7 +454,7 @@ again:
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
#ifdef SMP
/*
@ -2248,7 +2248,7 @@ init386(first)
/*
* We need this mutex before the console probe.
*/
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
/*
* Initialize the console before we print anything out.
@ -2263,7 +2263,7 @@ init386(first)
/*
* Giant is used early for at least debugger traps and unexpected traps.
*/
mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
#ifdef DDB
kdb_init();

1
sys/sys/kernel.h
View File

@ -113,6 +113,7 @@ enum sysinit_sub_id {
SI_SUB_VM = 0x1000000, /* virtual memory system init*/
SI_SUB_KMEM = 0x1800000, /* kernel memory*/
SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/
SI_SUB_MUTEX = 0x1A80000, /* mutex (witness) fixup */
SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */
SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */
SI_SUB_CPU = 0x2000000, /* CPU resource(s)*/

33
sys/sys/mutex.h
View File

@ -73,8 +73,7 @@
#define MTX_NOSWITCH 0x20 /* Do not switch on release */
#define MTX_FIRST 0x40 /* First spin lock holder */
#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */
#define MTX_COLD 0x100 /* Mutex init'd before malloc works */
#define MTX_QUIET 0x200 /* Don't log a mutex event */
#define MTX_QUIET 0x100 /* Don't log a mutex event */
/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */
#define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH)
@ -98,11 +97,13 @@ struct mtx_debug {
const char *mtxd_description;
};
#define mtx_description mtx_debug->mtxd_description
#define mtx_held mtx_debug->mtxd_held
#define mtx_line mtx_debug->mtxd_line
#define mtx_file mtx_debug->mtxd_file
#define mtx_witness mtx_debug->mtxd_witness
#define mtx_description mtx_union.mtxu_debug->mtxd_description
#define mtx_held mtx_union.mtxu_debug->mtxd_held
#define mtx_line mtx_union.mtxu_debug->mtxd_line
#define mtx_file mtx_union.mtxu_debug->mtxd_file
#define mtx_witness mtx_union.mtxu_debug->mtxd_witness
#else /* WITNESS */
#define mtx_description mtx_union.mtxu_description
#endif /* WITNESS */
/*
@ -112,25 +113,17 @@ struct mtx {
volatile uintptr_t mtx_lock; /* lock owner/gate/flags */
volatile u_int mtx_recurse; /* number of recursive holds */
u_int mtx_saveintr; /* saved flags (for spin locks) */
#ifdef WITNESS
struct mtx_debug *mtx_debug;
#else
const char *mtx_description;
#endif
int mtx_flags; /* flags passed to mtx_init() */
union {
struct mtx_debug *mtxu_debug;
const char *mtxu_description;
} mtx_union;
TAILQ_HEAD(, proc) mtx_blocked;
LIST_ENTRY(mtx) mtx_contested;
struct mtx *mtx_next; /* all locks in system */
struct mtx *mtx_prev;
};
#ifdef WITNESS
#define MUTEX_DECLARE(modifiers, name) \
static struct mtx_debug __mtx_debug_##name; \
modifiers struct mtx name = { 0, 0, 0, &__mtx_debug_##name }
#else
#define MUTEX_DECLARE(modifiers, name) modifiers struct mtx name
#endif
#define mp_fixme(string)
#ifdef _KERNEL