From d1c1b8413eda1667fac8797ca8ea32819f51eed3 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 21 Jan 2001 07:52:20 +0000 Subject: [PATCH] Remove MUTEX_DECLARE() and MTX_COLD. Instead, postpone full mutex initialization until after malloc() is safe to call, then iterate through all mutexes and complete their initialization. This change is necessary in order to avoid some circular bootstrapping dependencies. --- share/man/man9/Makefile | 1 - share/man/man9/mutex.9 | 40 +------------- sys/alpha/alpha/machdep.c | 8 +-- sys/amd64/amd64/machdep.c | 10 ++-- sys/amd64/amd64/tsc.c | 2 +- sys/amd64/isa/clock.c | 2 +- sys/i386/i386/machdep.c | 10 ++-- sys/i386/i386/tsc.c | 2 +- sys/i386/i386/vm86.c | 4 +- sys/i386/isa/clock.c | 2 +- sys/ia64/ia64/machdep.c | 8 +-- sys/isa/atrtc.c | 2 +- sys/kern/kern_lock.c | 6 +-- sys/kern/kern_malloc.c | 4 +- sys/kern/kern_mutex.c | 110 ++++++++++++++++++++++++++++---------- sys/kern/subr_turnstile.c | 110 ++++++++++++++++++++++++++++---------- sys/kern/subr_witness.c | 110 ++++++++++++++++++++++++++++---------- sys/pc98/cbus/clock.c | 2 +- sys/pc98/cbus/pcrtc.c | 2 +- sys/pc98/i386/machdep.c | 10 ++-- sys/pc98/pc98/clock.c | 2 +- sys/pc98/pc98/machdep.c | 10 ++-- sys/sys/kernel.h | 1 + sys/sys/mutex.h | 33 +++++------- 24 files changed, 301 insertions(+), 190 deletions(-) diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile index 9f15f79c0af..0b7fe36d944 100644 --- a/share/man/man9/Makefile +++ b/share/man/man9/Makefile @@ -148,7 +148,6 @@ MLINKS+=microtime.9 getnanotime.9 MLINKS+=microuptime.9 getmicrouptime.9 microuptime.9 nanouptime.9 MLINKS+=microuptime.9 getnanouptime.9 -MLINKS+=mutex.9 MUTEX_DECLARE.9 MLINKS+=mutex.9 mtx_init.9 MLINKS+=mutex.9 mtx_enter.9 MLINKS+=mutex.9 mtx_try_enter.9 diff --git a/share/man/man9/mutex.9 b/share/man/man9/mutex.9 index cdce7169ac9..dcba13509ff 100644 --- a/share/man/man9/mutex.9 +++ b/share/man/man9/mutex.9 @@ -39,8 +39,7 @@ .Nm mtx_exit , .Nm mtx_destroy , .Nm mtx_owned , -.Nm mtx_assert , -.Nm MUTEX_DECLARE +.Nm mtx_assert .Nd kernel synchronization primitives .Sh SYNOPSIS .Fd #include @@ -58,7 +57,6 @@ .Fn mtx_owned "struct mtx *mutex" .Ft void .Fn mtx_assert "struct mtx *mutex" "int what" -.Fn MUTEX_DECLARE "modifiers" "name" .Sh DESCRIPTION Mutexes are the most basic and primary method of process synchronization. The major design considerations for mutexes are: @@ -226,34 +224,6 @@ This assertion is only valid in conjuction with .Dv MA_OWNED . .El .Pp -The -.Fn MUTEX_DECLARE -macro is used to declare a mutex that is initialized before -.Xr malloc 9 -is operating. -Unfortunately, mutex initialization may require -.Xr malloc 9 . -However, some mutexes are initialized and used before -.Xr malloc 9 -can be used. -Declaring these mutexes with the -.Fn MUTEX_DECLARE -macro and then using the -.Dv MTX_COLD -flag when calling -.Fn mtx_init -allows these early mutexes to be initialized and used before -.Xr malloc 9 -is available. -The -.Ar modifiers -argument is a list of attributes to be applied to the mutex structure being -declared such as -.Dq static . -The -.Ar name -argument is the name of the mutex structure to declare. -.Pp The type of a mutex is not an attribute of the mutex, but instead a function of the .Fa flags @@ -398,14 +368,6 @@ This should be specified when it is known that the lock will usually remain unavailable for some time when it is not immediately available (i.e.: coarse grained locks protecting large subsystems). -.It Dv MTX_COLD -This option is only used in -.Fn mtx_init -and is used in conjunction with mutexes declared with -.Fn MUTEX_DECLARE -to initialize mutexes that are needed before -.Xr malloc 9 -is available for use. .It Dv MTX_QUIET This option is used to quiet logging messages during mutex operations. This can be used to trim superfluous logging messages for debugging purposes. diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index d220797d1aa..ecbc9fa2aec 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -154,8 +154,8 @@ struct bootinfo_kernel bootinfo; struct cpuhead cpuhead; -MUTEX_DECLARE( ,sched_lock); -MUTEX_DECLARE( ,Giant); +struct mtx sched_lock; +struct mtx Giant; struct user *proc0paddr; @@ -1003,8 +1003,8 @@ alpha_init(pfn, ptb, bim, bip, biv) /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); /* * Look at arguments passed to us and compute boothowto. diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 01cee5e074f..1bdcb0ec043 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -249,8 +249,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c index b34567b728c..14bcd933641 100644 --- a/sys/amd64/amd64/tsc.c +++ b/sys/amd64/amd64/tsc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c index b34567b728c..14bcd933641 100644 --- a/sys/amd64/isa/clock.c +++ b/sys/amd64/isa/clock.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 01cee5e074f..1bdcb0ec043 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -249,8 +249,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c index b34567b728c..14bcd933641 100644 --- a/sys/i386/i386/tsc.c +++ b/sys/i386/i386/tsc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/i386/i386/vm86.c b/sys/i386/i386/vm86.c index b081eef4a38..6e2f0806482 100644 --- a/sys/i386/i386/vm86.c +++ b/sys/i386/i386/vm86.c @@ -50,7 +50,7 @@ extern int i386_extend_pcb __P((struct proc *)); extern int vm86pa; extern struct pcb *vm86pcb; -MUTEX_DECLARE(static, vm86pcb_lock); +static struct mtx vm86pcb_lock; extern int vm86_bioscall(struct vm86frame *); extern void vm86_biosret(struct vm86frame *); @@ -426,7 +426,7 @@ vm86_initialize(void) pcb = &vml->vml_pcb; ext = &vml->vml_ext; - mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF | MTX_COLD); + mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF); bzero(pcb, sizeof(struct pcb)); pcb->new_ptd = vm86pa | PG_V | PG_RW | PG_U; diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c index b34567b728c..14bcd933641 100644 --- a/sys/i386/isa/clock.c +++ b/sys/i386/isa/clock.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index 30f5ef6fb8d..85bcd0b81dc 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -84,8 +84,8 @@ struct bootinfo_kernel bootinfo; struct cpuhead cpuhead; -MUTEX_DECLARE( ,sched_lock); -MUTEX_DECLARE( ,Giant); +struct mtx sched_lock; +struct mtx Giant; struct user *proc0paddr; @@ -590,8 +590,8 @@ ia64_init() /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #if 0 /* diff --git a/sys/isa/atrtc.c b/sys/isa/atrtc.c index b34567b728c..14bcd933641 100644 --- a/sys/isa/atrtc.c +++ b/sys/isa/atrtc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 158377d3cad..8314a0e9c2d 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -80,7 +80,7 @@ extern int lock_nmtx; int lock_mtx_selector; struct mtx *lock_mtx_array; -MUTEX_DECLARE(static, lock_mtx); +static struct mtx lock_mtx; static int acquire(struct lock *lkp, int extflags, int wanted); static int apause(struct lock *lkp, int flags); @@ -98,7 +98,7 @@ lockmgr_init(void *dummy __unused) * initialized in a call to lockinit(). */ if (lock_mtx_selector == 0) - mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD); + mtx_init(&lock_mtx, "lockmgr", MTX_DEF); else { /* * This is necessary if (lock_nmtx == 1) and doesn't hurt @@ -546,7 +546,7 @@ lockinit(lkp, prio, wmesg, timo, flags) * so there's no reason to protect modification of * lock_mtx_selector or lock_mtx. */ - mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD); + mtx_init(&lock_mtx, "lockmgr", MTX_DEF); lock_mtx_selector = 1; } lkp->lk_interlock = &lock_mtx; diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index d6275b8859a..396f02bbb14 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -74,7 +74,7 @@ static struct kmemusage *kmemusage; static char *kmembase; static char *kmemlimit; -MUTEX_DECLARE(static, malloc_mtx); +static struct mtx malloc_mtx; u_int vm_kmem_size; @@ -439,7 +439,7 @@ kmeminit(dummy) #error "kmeminit: MAXALLOCSAVE too small" #endif - mtx_init(&malloc_mtx, "malloc", MTX_DEF | MTX_COLD); + mtx_init(&malloc_mtx, "malloc", MTX_DEF); /* * Try to auto-tune the kernel memory size, so that it is diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/pc98/cbus/clock.c b/sys/pc98/cbus/clock.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/cbus/clock.c +++ b/sys/pc98/cbus/clock.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/cbus/pcrtc.c b/sys/pc98/cbus/pcrtc.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/cbus/pcrtc.c +++ b/sys/pc98/cbus/pcrtc.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/i386/machdep.c b/sys/pc98/i386/machdep.c index cf9df15ac94..410532461ec 100644 --- a/sys/pc98/i386/machdep.c +++ b/sys/pc98/i386/machdep.c @@ -262,8 +262,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -454,7 +454,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -2248,7 +2248,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2263,7 +2263,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/pc98/pc98/clock.c b/sys/pc98/pc98/clock.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/pc98/clock.c +++ b/sys/pc98/pc98/clock.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index cf9df15ac94..410532461ec 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -262,8 +262,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -454,7 +454,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -2248,7 +2248,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2263,7 +2263,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h index 48acdffeba1..3b1be03c587 100644 --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -113,6 +113,7 @@ enum sysinit_sub_id { SI_SUB_VM = 0x1000000, /* virtual memory system init*/ SI_SUB_KMEM = 0x1800000, /* kernel memory*/ SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/ + SI_SUB_MUTEX = 0x1A80000, /* mutex (witness) fixup */ SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */ SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */ SI_SUB_CPU = 0x2000000, /* CPU resource(s)*/ diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index e4660ec539f..4ce9d36e165 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -73,8 +73,7 @@ #define MTX_NOSWITCH 0x20 /* Do not switch on release */ #define MTX_FIRST 0x40 /* First spin lock holder */ #define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ -#define MTX_COLD 0x100 /* Mutex init'd before malloc works */ -#define MTX_QUIET 0x200 /* Don't log a mutex event */ +#define MTX_QUIET 0x100 /* Don't log a mutex event */ /* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ #define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) @@ -98,11 +97,13 @@ struct mtx_debug { const char *mtxd_description; }; -#define mtx_description mtx_debug->mtxd_description -#define mtx_held mtx_debug->mtxd_held -#define mtx_line mtx_debug->mtxd_line -#define mtx_file mtx_debug->mtxd_file -#define mtx_witness mtx_debug->mtxd_witness +#define mtx_description mtx_union.mtxu_debug->mtxd_description +#define mtx_held mtx_union.mtxu_debug->mtxd_held +#define mtx_line mtx_union.mtxu_debug->mtxd_line +#define mtx_file mtx_union.mtxu_debug->mtxd_file +#define mtx_witness mtx_union.mtxu_debug->mtxd_witness +#else /* WITNESS */ +#define mtx_description mtx_union.mtxu_description #endif /* WITNESS */ /* @@ -112,25 +113,17 @@ struct mtx { volatile uintptr_t mtx_lock; /* lock owner/gate/flags */ volatile u_int mtx_recurse; /* number of recursive holds */ u_int mtx_saveintr; /* saved flags (for spin locks) */ -#ifdef WITNESS - struct mtx_debug *mtx_debug; -#else - const char *mtx_description; -#endif + int mtx_flags; /* flags passed to mtx_init() */ + union { + struct mtx_debug *mtxu_debug; + const char *mtxu_description; + } mtx_union; TAILQ_HEAD(, proc) mtx_blocked; LIST_ENTRY(mtx) mtx_contested; struct mtx *mtx_next; /* all locks in system */ struct mtx *mtx_prev; }; -#ifdef WITNESS -#define MUTEX_DECLARE(modifiers, name) \ - static struct mtx_debug __mtx_debug_##name; \ - modifiers struct mtx name = { 0, 0, 0, &__mtx_debug_##name } -#else -#define MUTEX_DECLARE(modifiers, name) modifiers struct mtx name -#endif - #define mp_fixme(string) #ifdef _KERNEL