diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile index 9f15f79c0af..0b7fe36d944 100644 --- a/share/man/man9/Makefile +++ b/share/man/man9/Makefile @@ -148,7 +148,6 @@ MLINKS+=microtime.9 getnanotime.9 MLINKS+=microuptime.9 getmicrouptime.9 microuptime.9 nanouptime.9 MLINKS+=microuptime.9 getnanouptime.9 -MLINKS+=mutex.9 MUTEX_DECLARE.9 MLINKS+=mutex.9 mtx_init.9 MLINKS+=mutex.9 mtx_enter.9 MLINKS+=mutex.9 mtx_try_enter.9 diff --git a/share/man/man9/mutex.9 b/share/man/man9/mutex.9 index cdce7169ac9..dcba13509ff 100644 --- a/share/man/man9/mutex.9 +++ b/share/man/man9/mutex.9 @@ -39,8 +39,7 @@ .Nm mtx_exit , .Nm mtx_destroy , .Nm mtx_owned , -.Nm mtx_assert , -.Nm MUTEX_DECLARE +.Nm mtx_assert .Nd kernel synchronization primitives .Sh SYNOPSIS .Fd #include @@ -58,7 +57,6 @@ .Fn mtx_owned "struct mtx *mutex" .Ft void .Fn mtx_assert "struct mtx *mutex" "int what" -.Fn MUTEX_DECLARE "modifiers" "name" .Sh DESCRIPTION Mutexes are the most basic and primary method of process synchronization. The major design considerations for mutexes are: @@ -226,34 +224,6 @@ This assertion is only valid in conjuction with .Dv MA_OWNED . .El .Pp -The -.Fn MUTEX_DECLARE -macro is used to declare a mutex that is initialized before -.Xr malloc 9 -is operating. -Unfortunately, mutex initialization may require -.Xr malloc 9 . -However, some mutexes are initialized and used before -.Xr malloc 9 -can be used. -Declaring these mutexes with the -.Fn MUTEX_DECLARE -macro and then using the -.Dv MTX_COLD -flag when calling -.Fn mtx_init -allows these early mutexes to be initialized and used before -.Xr malloc 9 -is available. -The -.Ar modifiers -argument is a list of attributes to be applied to the mutex structure being -declared such as -.Dq static . -The -.Ar name -argument is the name of the mutex structure to declare. -.Pp The type of a mutex is not an attribute of the mutex, but instead a function of the .Fa flags @@ -398,14 +368,6 @@ This should be specified when it is known that the lock will usually remain unavailable for some time when it is not immediately available (i.e.: coarse grained locks protecting large subsystems). -.It Dv MTX_COLD -This option is only used in -.Fn mtx_init -and is used in conjunction with mutexes declared with -.Fn MUTEX_DECLARE -to initialize mutexes that are needed before -.Xr malloc 9 -is available for use. .It Dv MTX_QUIET This option is used to quiet logging messages during mutex operations. This can be used to trim superfluous logging messages for debugging purposes. diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index d220797d1aa..ecbc9fa2aec 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -154,8 +154,8 @@ struct bootinfo_kernel bootinfo; struct cpuhead cpuhead; -MUTEX_DECLARE( ,sched_lock); -MUTEX_DECLARE( ,Giant); +struct mtx sched_lock; +struct mtx Giant; struct user *proc0paddr; @@ -1003,8 +1003,8 @@ alpha_init(pfn, ptb, bim, bip, biv) /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); /* * Look at arguments passed to us and compute boothowto. diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 01cee5e074f..1bdcb0ec043 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -249,8 +249,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c index b34567b728c..14bcd933641 100644 --- a/sys/amd64/amd64/tsc.c +++ b/sys/amd64/amd64/tsc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c index b34567b728c..14bcd933641 100644 --- a/sys/amd64/isa/clock.c +++ b/sys/amd64/isa/clock.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 01cee5e074f..1bdcb0ec043 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -249,8 +249,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c index b34567b728c..14bcd933641 100644 --- a/sys/i386/i386/tsc.c +++ b/sys/i386/i386/tsc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/i386/i386/vm86.c b/sys/i386/i386/vm86.c index b081eef4a38..6e2f0806482 100644 --- a/sys/i386/i386/vm86.c +++ b/sys/i386/i386/vm86.c @@ -50,7 +50,7 @@ extern int i386_extend_pcb __P((struct proc *)); extern int vm86pa; extern struct pcb *vm86pcb; -MUTEX_DECLARE(static, vm86pcb_lock); +static struct mtx vm86pcb_lock; extern int vm86_bioscall(struct vm86frame *); extern void vm86_biosret(struct vm86frame *); @@ -426,7 +426,7 @@ vm86_initialize(void) pcb = &vml->vml_pcb; ext = &vml->vml_ext; - mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF | MTX_COLD); + mtx_init(&vm86pcb_lock, "vm86pcb lock", MTX_DEF); bzero(pcb, sizeof(struct pcb)); pcb->new_ptd = vm86pa | PG_V | PG_RW | PG_U; diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c index b34567b728c..14bcd933641 100644 --- a/sys/i386/isa/clock.c +++ b/sys/i386/isa/clock.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index 30f5ef6fb8d..85bcd0b81dc 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -84,8 +84,8 @@ struct bootinfo_kernel bootinfo; struct cpuhead cpuhead; -MUTEX_DECLARE( ,sched_lock); -MUTEX_DECLARE( ,Giant); +struct mtx sched_lock; +struct mtx Giant; struct user *proc0paddr; @@ -590,8 +590,8 @@ ia64_init() /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #if 0 /* diff --git a/sys/isa/atrtc.c b/sys/isa/atrtc.c index b34567b728c..14bcd933641 100644 --- a/sys/isa/atrtc.c +++ b/sys/isa/atrtc.c @@ -140,7 +140,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 158377d3cad..8314a0e9c2d 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -80,7 +80,7 @@ extern int lock_nmtx; int lock_mtx_selector; struct mtx *lock_mtx_array; -MUTEX_DECLARE(static, lock_mtx); +static struct mtx lock_mtx; static int acquire(struct lock *lkp, int extflags, int wanted); static int apause(struct lock *lkp, int flags); @@ -98,7 +98,7 @@ lockmgr_init(void *dummy __unused) * initialized in a call to lockinit(). */ if (lock_mtx_selector == 0) - mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD); + mtx_init(&lock_mtx, "lockmgr", MTX_DEF); else { /* * This is necessary if (lock_nmtx == 1) and doesn't hurt @@ -546,7 +546,7 @@ lockinit(lkp, prio, wmesg, timo, flags) * so there's no reason to protect modification of * lock_mtx_selector or lock_mtx. */ - mtx_init(&lock_mtx, "lockmgr", MTX_DEF | MTX_COLD); + mtx_init(&lock_mtx, "lockmgr", MTX_DEF); lock_mtx_selector = 1; } lkp->lk_interlock = &lock_mtx; diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index d6275b8859a..396f02bbb14 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -74,7 +74,7 @@ static struct kmemusage *kmemusage; static char *kmembase; static char *kmemlimit; -MUTEX_DECLARE(static, malloc_mtx); +static struct mtx malloc_mtx; u_int vm_kmem_size; @@ -439,7 +439,7 @@ kmeminit(dummy) #error "kmeminit: MAXALLOCSAVE too small" #endif - mtx_init(&malloc_mtx, "malloc", MTX_DEF | MTX_COLD); + mtx_init(&malloc_mtx, "malloc", MTX_DEF); /* * Try to auto-tune the kernel memory size, so that it is diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 0d42519a195..d99645939ca 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -88,11 +88,16 @@ #ifdef WITNESS static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0, "All mutexes queue head" }; -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, &all_mtx_debug, +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {&all_mtx_debug}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; +/* + * Set to 0 once mutexes have been fully initialized so that witness code can be + * safely executed. + */ +static int witness_cold = 1; #else /* WITNESS */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, "All mutexes queue head", +static struct mtx all_mtx = { 0, MTX_UNOWNED, 0, 0, {"All mutexes queue head"}, TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL }, &all_mtx, &all_mtx }; #endif /* WITNESS */ @@ -521,6 +526,10 @@ mtx_validate(struct mtx *m, int when) int i; int retval = 0; +#ifdef WITNESS + if (witness_cold) + return 0; +#endif if (m == &all_mtx || cold) return 0; @@ -576,39 +585,35 @@ mtx_validate(struct mtx *m, int when) void mtx_init(struct mtx *m, const char *t, int flag) { -#ifdef WITNESS - struct mtx_debug *debug; -#endif - if ((flag & MTX_QUIET) == 0) CTR2(KTR_LOCK, "mtx_init 0x%p (%s)", m, t); #ifdef MUTEX_DEBUG if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ return; #endif -#ifdef WITNESS - if (flag & MTX_COLD) - debug = m->mtx_debug; - else - debug = NULL; - if (debug == NULL) { -#ifdef DIAGNOSTIC - if(cold && bootverbose) - printf("malloc'ing mtx_debug while cold for %s\n", t); -#endif - /* XXX - should not use DEVBUF */ - debug = malloc(sizeof(struct mtx_debug), M_DEVBUF, - M_NOWAIT | M_ZERO); - MPASS(debug != NULL); - } -#endif bzero((void *)m, sizeof *m); TAILQ_INIT(&m->mtx_blocked); #ifdef WITNESS - m->mtx_debug = debug; -#endif + if (!witness_cold) { + /* XXX - should not use DEVBUF */ + m->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(m->mtx_union.mtxu_debug != NULL); + + m->mtx_description = t; + } else { + /* + * Save a pointer to the description so that witness_fixup() + * can properly initialize this mutex later on. + */ + m->mtx_union.mtxu_description = t; + } +#else m->mtx_description = t; +#endif + + m->mtx_flags = flag; m->mtx_lock = MTX_UNOWNED; /* Put on all mutex queue */ mtx_enter(&all_mtx, MTX_DEF); @@ -619,13 +624,20 @@ mtx_init(struct mtx *m, const char *t, int flag) if (++mtx_cur_cnt > mtx_max_cnt) mtx_max_cnt = mtx_cur_cnt; mtx_exit(&all_mtx, MTX_DEF); - witness_init(m, flag); +#ifdef WITNESS + if (!witness_cold) + witness_init(m, flag); +#endif } void mtx_destroy(struct mtx *m) { +#ifdef WITNESS + KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", + __FUNCTION__)); +#endif CTR2(KTR_LOCK, "mtx_destroy 0x%p (%s)", m, m->mtx_description); #ifdef MUTEX_DEBUG if (m->mtx_next == NULL) @@ -653,13 +665,40 @@ mtx_destroy(struct mtx *m) m->mtx_next = m->mtx_prev = NULL; #endif #ifdef WITNESS - free(m->mtx_debug, M_DEVBUF); - m->mtx_debug = NULL; + free(m->mtx_union.mtxu_debug, M_DEVBUF); + m->mtx_union.mtxu_debug = NULL; #endif mtx_cur_cnt--; mtx_exit(&all_mtx, MTX_DEF); } +static void +witness_fixup(void *dummy __unused) +{ +#ifdef WITNESS + struct mtx *mp; + const char *description; + + /* Iterate through all mutexes and finish up mutex initialization. */ + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + description = mp->mtx_union.mtxu_description; + + /* XXX - should not use DEVBUF */ + mp->mtx_union.mtxu_debug = malloc(sizeof(struct mtx_debug), + M_DEVBUF, M_NOWAIT | M_ZERO); + MPASS(mp->mtx_union.mtxu_debug != NULL); + + mp->mtx_description = description; + + witness_init(mp, mp->mtx_flags); + } + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); +#endif +} +SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) + /* * The non-inlined versions of the mtx_*() functions are always built (above), * but the witness code depends on the WITNESS kernel option being specified. @@ -716,7 +755,7 @@ int witness_skipspin = 0; SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -MUTEX_DECLARE(static,w_mtx); +static struct mtx w_mtx; static struct witness *w_free; static struct witness *w_all; static int w_inited; @@ -813,6 +852,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) int go_into_ddb = 0; #endif /* DDB */ + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -957,6 +998,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + if (witness_cold) + return; if (panicstr) return; w = m->mtx_witness; @@ -1003,6 +1046,8 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) struct proc *p; struct witness *w = m->mtx_witness; + if (witness_cold) + return; if (panicstr) return; if (flags & MTX_SPIN) { @@ -1053,6 +1098,7 @@ witness_display(void(*prnt)(const char *fmt, ...)) { struct witness *w, *w1; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); witness_levelall(); for (w = w_all; w; w = w->w_next) { @@ -1085,6 +1131,7 @@ witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) char **sleep; int n = 0; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); p = CURPROC; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1122,7 +1169,7 @@ enroll(const char *description, int flag) return (NULL); if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_COLD | MTX_SPIN); + mtx_init(&w_mtx, "witness lock", MTX_SPIN); for (i = 0; i < WITNESS_COUNT; i++) { w = &w_data[i]; witness_free(w); @@ -1401,6 +1448,7 @@ witness_list(struct proc *p) struct mtx *m; int nheld; + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); nheld = 0; for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; m = LIST_NEXT(m, mtx_held)) { @@ -1416,6 +1464,8 @@ witness_list(struct proc *p) void witness_save(struct mtx *m, const char **filep, int *linep) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); *filep = m->mtx_witness->w_file; *linep = m->mtx_witness->w_line; } @@ -1423,6 +1473,8 @@ witness_save(struct mtx *m, const char **filep, int *linep) void witness_restore(struct mtx *m, const char *file, int line) { + + KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); m->mtx_witness->w_file = file; m->mtx_witness->w_line = line; } diff --git a/sys/pc98/cbus/clock.c b/sys/pc98/cbus/clock.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/cbus/clock.c +++ b/sys/pc98/cbus/clock.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/cbus/pcrtc.c b/sys/pc98/cbus/pcrtc.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/cbus/pcrtc.c +++ b/sys/pc98/cbus/pcrtc.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/i386/machdep.c b/sys/pc98/i386/machdep.c index cf9df15ac94..410532461ec 100644 --- a/sys/pc98/i386/machdep.c +++ b/sys/pc98/i386/machdep.c @@ -262,8 +262,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -454,7 +454,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -2248,7 +2248,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2263,7 +2263,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/pc98/pc98/clock.c b/sys/pc98/pc98/clock.c index 3acf9a408cf..7de923d1c5a 100644 --- a/sys/pc98/pc98/clock.c +++ b/sys/pc98/pc98/clock.c @@ -154,7 +154,7 @@ int timer0_max_count; u_int tsc_freq; int tsc_is_broken; int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ -MUTEX_DECLARE(,clock_lock); +struct mtx clock_lock; static int beeping = 0; static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index cf9df15ac94..410532461ec 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -262,8 +262,8 @@ static struct globaldata __globaldata; struct cpuhead cpuhead; -MUTEX_DECLARE(,sched_lock); -MUTEX_DECLARE(,Giant); +struct mtx sched_lock; +struct mtx Giant; static void cpu_startup(dummy) @@ -454,7 +454,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); #ifdef SMP /* @@ -2248,7 +2248,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2263,7 +2263,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h index 48acdffeba1..3b1be03c587 100644 --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -113,6 +113,7 @@ enum sysinit_sub_id { SI_SUB_VM = 0x1000000, /* virtual memory system init*/ SI_SUB_KMEM = 0x1800000, /* kernel memory*/ SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/ + SI_SUB_MUTEX = 0x1A80000, /* mutex (witness) fixup */ SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */ SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */ SI_SUB_CPU = 0x2000000, /* CPU resource(s)*/ diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index e4660ec539f..4ce9d36e165 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -73,8 +73,7 @@ #define MTX_NOSWITCH 0x20 /* Do not switch on release */ #define MTX_FIRST 0x40 /* First spin lock holder */ #define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ -#define MTX_COLD 0x100 /* Mutex init'd before malloc works */ -#define MTX_QUIET 0x200 /* Don't log a mutex event */ +#define MTX_QUIET 0x100 /* Don't log a mutex event */ /* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ #define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) @@ -98,11 +97,13 @@ struct mtx_debug { const char *mtxd_description; }; -#define mtx_description mtx_debug->mtxd_description -#define mtx_held mtx_debug->mtxd_held -#define mtx_line mtx_debug->mtxd_line -#define mtx_file mtx_debug->mtxd_file -#define mtx_witness mtx_debug->mtxd_witness +#define mtx_description mtx_union.mtxu_debug->mtxd_description +#define mtx_held mtx_union.mtxu_debug->mtxd_held +#define mtx_line mtx_union.mtxu_debug->mtxd_line +#define mtx_file mtx_union.mtxu_debug->mtxd_file +#define mtx_witness mtx_union.mtxu_debug->mtxd_witness +#else /* WITNESS */ +#define mtx_description mtx_union.mtxu_description #endif /* WITNESS */ /* @@ -112,25 +113,17 @@ struct mtx { volatile uintptr_t mtx_lock; /* lock owner/gate/flags */ volatile u_int mtx_recurse; /* number of recursive holds */ u_int mtx_saveintr; /* saved flags (for spin locks) */ -#ifdef WITNESS - struct mtx_debug *mtx_debug; -#else - const char *mtx_description; -#endif + int mtx_flags; /* flags passed to mtx_init() */ + union { + struct mtx_debug *mtxu_debug; + const char *mtxu_description; + } mtx_union; TAILQ_HEAD(, proc) mtx_blocked; LIST_ENTRY(mtx) mtx_contested; struct mtx *mtx_next; /* all locks in system */ struct mtx *mtx_prev; }; -#ifdef WITNESS -#define MUTEX_DECLARE(modifiers, name) \ - static struct mtx_debug __mtx_debug_##name; \ - modifiers struct mtx name = { 0, 0, 0, &__mtx_debug_##name } -#else -#define MUTEX_DECLARE(modifiers, name) modifiers struct mtx name -#endif - #define mp_fixme(string) #ifdef _KERNEL