From da7bbd2c08cb22313872fe600b981466c9af1566 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 5 Aug 2008 20:02:31 +0000 Subject: [PATCH] If a thread that is swapped out is made runnable, then the setrunnable() routine wakes up proc0 so that proc0 can swap the thread back in. Historically, this has been done by waking up proc0 directly from setrunnable() itself via a wakeup(). When waking up a sleeping thread that was swapped out (the usual case when waking proc0 since only sleeping threads are eligible to be swapped out), this resulted in a bit of recursion (e.g. wakeup() -> setrunnable() -> wakeup()). With sleep queues having separate locks in 6.x and later, this caused a spin lock LOR (sleepq lock -> sched_lock/thread lock -> sleepq lock). An attempt was made to fix this in 7.0 by making the proc0 wakeup use the ithread mechanism for doing the wakeup. However, this required grabbing proc0's thread lock to perform the wakeup. If proc0 was asleep elsewhere in the kernel (e.g. waiting for disk I/O), then this degenerated into the same LOR since the thread lock would be some other sleepq lock. Fix this by deferring the wakeup of the swapper until after the sleepq lock held by the upper layer has been locked. The setrunnable() routine now returns a boolean value to indicate whether or not proc0 needs to be woken up. The end result is that consumers of the sleepq API such as *sleep/wakeup, condition variables, sx locks, and lockmgr, have to wakeup proc0 if they get a non-zero return value from sleepq_abort(), sleepq_broadcast(), or sleepq_signal(). Discussed with: jeff Glanced at by: sam Tested by: Jurgen Weber jurgen - ish com au MFC after: 2 weeks --- sys/kern/kern_condvar.c | 13 ++++++-- sys/kern/kern_lock.c | 25 +++++++++----- sys/kern/kern_sig.c | 12 +++++-- sys/kern/kern_sx.c | 23 +++++++++---- sys/kern/kern_synch.c | 32 +++++++++--------- sys/kern/kern_thread.c | 14 ++++++-- sys/kern/subr_sleepqueue.c | 67 ++++++++++++++++++++++++++++---------- sys/sys/proc.h | 2 +- sys/sys/sleepqueue.h | 6 ++-- sys/vm/vm_glue.c | 51 ++++++++--------------------- 10 files changed, 149 insertions(+), 96 deletions(-) diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c index eb804ca5eb4a..5ee40a32bec6 100644 --- a/sys/kern/kern_condvar.c +++ b/sys/kern/kern_condvar.c @@ -389,13 +389,17 @@ _cv_timedwait_sig(struct cv *cvp, struct lock_object *lock, int timo) void cv_signal(struct cv *cvp) { + int wakeup_swapper; + wakeup_swapper = 0; sleepq_lock(cvp); if (cvp->cv_waiters > 0) { cvp->cv_waiters--; - sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); + wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); } sleepq_release(cvp); + if (wakeup_swapper) + kick_proc0(); } /* @@ -405,16 +409,21 @@ cv_signal(struct cv *cvp) void cv_broadcastpri(struct cv *cvp, int pri) { + int wakeup_swapper; + /* * XXX sleepq_broadcast pri argument changed from -1 meaning * no pri to 0 meaning no pri. */ + wakeup_swapper = 0; if (pri == -1) pri = 0; sleepq_lock(cvp); if (cvp->cv_waiters > 0) { cvp->cv_waiters = 0; - sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); + wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); } sleepq_release(cvp); + if (wakeup_swapper) + kick_proc0(); } diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index b3f0e25847fe..2f71efa45574 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -196,17 +196,18 @@ sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk, return (error); } -static __inline void +static __inline int wakeupshlk(struct lock *lk, const char *file, int line) { uintptr_t v, x; - int queue; + int queue, wakeup_swapper; TD_LOCKS_DEC(curthread); TD_SLOCKS_DEC(curthread); WITNESS_UNLOCK(&lk->lock_object, 0, file, line); LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); + wakeup_swapper = 0; for (;;) { x = lk->lk_lock; @@ -261,12 +262,14 @@ wakeupshlk(struct lock *lk, const char *file, int line) LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); - sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); + wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, + 0, queue); sleepq_release(&lk->lock_object); break; } lock_profile_release_lock(&lk->lock_object); + return (wakeup_swapper); } static void @@ -335,7 +338,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, const char *iwmesg; uintptr_t tid, v, x; u_int op; - int contested, error, ipri, itimo, queue; + int contested, error, ipri, itimo, queue, wakeup_swapper; contested = 0; error = 0; @@ -367,6 +370,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE)) op = LK_EXCLUSIVE; + wakeup_swapper = 0; switch (op) { case LK_SHARED: if (LK_CAN_WITNESS(flags)) @@ -495,7 +499,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, * We have been unable to succeed in upgrading, so just * give up the shared lock. */ - wakeupshlk(lk, file, line); + wakeup_swapper += wakeupshlk(lk, file, line); /* FALLTHROUGH */ case LK_EXCLUSIVE: @@ -710,11 +714,12 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); atomic_store_rel_ptr(&lk->lk_lock, v); - sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); + wakeup_swapper = sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, queue); sleepq_release(&lk->lock_object); break; } else - wakeupshlk(lk, file, line); + wakeup_swapper = wakeupshlk(lk, file, line); break; case LK_DRAIN: if (LK_CAN_WITNESS(flags)) @@ -782,8 +787,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, "%s: %p waking up all threads on the %s queue", __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); - sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, - 0, queue); + wakeup_swapper += sleepq_broadcast( + &lk->lock_object, SLEEPQ_LK, 0, queue); /* * If shared waiters have been woken up we need @@ -850,6 +855,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, if (flags & LK_INTERLOCK) class->lc_unlock(ilk); + if (wakeup_swapper) + kick_proc0(); return (error); } diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c index ff02ee3cb94b..00c29d14af01 100644 --- a/sys/kern/kern_sig.c +++ b/sys/kern/kern_sig.c @@ -1957,6 +1957,7 @@ tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi) struct sigacts *ps; int intrval; int ret = 0; + int wakeup_swapper; PROC_LOCK_ASSERT(p, MA_OWNED); @@ -2165,12 +2166,15 @@ tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi) * the PROCESS runnable, leave it stopped. * It may run a bit until it hits a thread_suspend_check(). */ + wakeup_swapper = 0; PROC_SLOCK(p); thread_lock(td); if (TD_ON_SLEEPQ(td) && (td->td_flags & TDF_SINTR)) - sleepq_abort(td, intrval); + wakeup_swapper = sleepq_abort(td, intrval); thread_unlock(td); PROC_SUNLOCK(p); + if (wakeup_swapper) + kick_proc0(); goto out; /* * Mutexes are short lived. Threads waiting on them will @@ -2237,7 +2241,9 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval) { struct proc *p = td->td_proc; register int prop; + int wakeup_swapper; + wakeup_swapper = 0; PROC_LOCK_ASSERT(p, MA_OWNED); prop = sigprop(sig); @@ -2281,7 +2287,7 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval) if (td->td_priority > PUSER) sched_prio(td, PUSER); - sleepq_abort(td, intrval); + wakeup_swapper = sleepq_abort(td, intrval); } else { /* * Other states do nothing with the signal immediately, @@ -2296,6 +2302,8 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval) out: PROC_SUNLOCK(p); thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); } static void diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index 1f0011c813a5..5006793a168b 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -361,6 +361,7 @@ void _sx_downgrade(struct sx *sx, const char *file, int line) { uintptr_t x; + int wakeup_swapper; KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); @@ -401,15 +402,19 @@ _sx_downgrade(struct sx *sx, const char *file, int line) * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single * shared lock. If there are any shared waiters, wake them up. */ + wakeup_swapper = 0; x = sx->sx_lock; atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS)); if (x & SX_LOCK_SHARED_WAITERS) - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, - SQ_SHARED_QUEUE); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, + 0, SQ_SHARED_QUEUE); sleepq_release(&sx->lock_object); LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); + + if (wakeup_swapper) + kick_proc0(); } /* @@ -589,7 +594,7 @@ void _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) { uintptr_t x; - int queue; + int queue, wakeup_swapper; MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); @@ -627,8 +632,11 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); atomic_store_rel_ptr(&sx->sx_lock, x); - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, queue); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, + queue); sleepq_release(&sx->lock_object); + if (wakeup_swapper) + kick_proc0(); } /* @@ -795,6 +803,7 @@ void _sx_sunlock_hard(struct sx *sx, const char *file, int line) { uintptr_t x; + int wakeup_swapper; for (;;) { x = sx->sx_lock; @@ -862,9 +871,11 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line) if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p waking up all thread on" "exclusive queue", __func__, sx); - sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, - SQ_EXCLUSIVE_QUEUE); + wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, + 0, SQ_EXCLUSIVE_QUEUE); sleepq_release(&sx->lock_object); + if (wakeup_swapper) + kick_proc0(); break; } } diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index c322ace3970f..6c10ba4c1ff2 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -327,10 +327,13 @@ pause(const char *wmesg, int timo) void wakeup(void *ident) { + int wakeup_swapper; sleepq_lock(ident); - sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); + wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); sleepq_release(ident); + if (wakeup_swapper) + kick_proc0(); } /* @@ -341,10 +344,13 @@ wakeup(void *ident) void wakeup_one(void *ident) { + int wakeup_swapper; sleepq_lock(ident); - sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); + wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); sleepq_release(ident); + if (wakeup_swapper) + kick_proc0(); } static void @@ -440,11 +446,11 @@ mi_switch(int flags, struct thread *newtd) } /* - * Change process state to be runnable, - * placing it on the run queue if it is in memory, - * and awakening the swapper if it isn't in memory. + * Change thread state to be runnable, placing it on the run queue if + * it is in memory. If it is swapped out, return true so our caller + * will know to awaken the swapper. */ -void +int setrunnable(struct thread *td) { @@ -454,15 +460,15 @@ setrunnable(struct thread *td) switch (td->td_state) { case TDS_RUNNING: case TDS_RUNQ: - return; + return (0); case TDS_INHIBITED: /* * If we are only inhibited because we are swapped out * then arange to swap in this process. Otherwise just return. */ if (td->td_inhibitors != TDI_SWAPPED) - return; - /* XXX: intentional fall-through ? */ + return (0); + /* FALLTHROUGH */ case TDS_CAN_RUN: break; default: @@ -472,15 +478,11 @@ setrunnable(struct thread *td) if ((td->td_flags & TDF_INMEM) == 0) { if ((td->td_flags & TDF_SWAPINREQ) == 0) { td->td_flags |= TDF_SWAPINREQ; - /* - * due to a LOR between the thread lock and - * the sleepqueue chain locks, use - * lower level scheduling functions. - */ - kick_proc0(); + return (1); } } else sched_wakeup(td); + return (0); } /* diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index a3d5da72f53e..6b4ea6f004af 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -508,7 +508,7 @@ thread_single(int mode) struct thread *td; struct thread *td2; struct proc *p; - int remaining; + int remaining, wakeup_swapper; td = curthread; p = td->td_proc; @@ -545,6 +545,7 @@ thread_single(int mode) while (remaining != 1) { if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) goto stopme; + wakeup_swapper = 0; FOREACH_THREAD_IN_PROC(p, td2) { if (td2 == td) continue; @@ -559,7 +560,8 @@ thread_single(int mode) thread_unsuspend_one(td2); if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) - sleepq_abort(td2, EINTR); + wakeup_swapper = + sleepq_abort(td2, EINTR); break; case SINGLE_BOUNDARY: break; @@ -585,6 +587,8 @@ thread_single(int mode) #endif thread_unlock(td2); } + if (wakeup_swapper) + kick_proc0(); if (mode == SINGLE_EXIT) remaining = p->p_numthreads; else if (mode == SINGLE_BOUNDARY) @@ -787,7 +791,11 @@ thread_unsuspend_one(struct thread *td) KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); TD_CLR_SUSPENDED(td); p->p_suspcount--; - setrunnable(td); + if (setrunnable(td)) { +#ifdef INVARIANTS + panic("not waking up swapper"); +#endif + } } /* diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c index 3878d812d93a..313786aace1e 100644 --- a/sys/kern/subr_sleepqueue.c +++ b/sys/kern/subr_sleepqueue.c @@ -160,7 +160,7 @@ static int sleepq_check_timeout(void); static void sleepq_dtor(void *mem, int size, void *arg); #endif static int sleepq_init(void *mem, int size, int flags); -static void sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, +static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri); static void sleepq_switch(void *wchan, int pri); static void sleepq_timeout(void *arg); @@ -434,7 +434,15 @@ sleepq_catch_signals(void *wchan, int pri) */ if (TD_ON_SLEEPQ(td)) { sq = sleepq_lookup(wchan); - sleepq_resume_thread(sq, td, 0); + if (sleepq_resume_thread(sq, td, 0)) { +#ifdef INVARIANTS + /* + * This thread hasn't gone to sleep yet, so it + * should not be swapped out. + */ + panic("not waking up swapper"); +#endif + } } mtx_unlock_spin(&sc->sc_lock); MPASS(td->td_lock != &sc->sc_lock); @@ -474,7 +482,15 @@ sleepq_switch(void *wchan, int pri) if (td->td_flags & TDF_TIMEOUT) { MPASS(TD_ON_SLEEPQ(td)); sq = sleepq_lookup(wchan); - sleepq_resume_thread(sq, td, 0); + if (sleepq_resume_thread(sq, td, 0)) { +#ifdef INVARIANTS + /* + * This thread hasn't gone to sleep yet, so it + * should not be swapped out. + */ + panic("not waking up swapper"); +#endif + } mtx_unlock_spin(&sc->sc_lock); return; } @@ -631,7 +647,7 @@ sleepq_timedwait_sig(void *wchan, int pri) * Removes a thread from a sleep queue and makes it * runnable. */ -static void +static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri) { struct sleepqueue_chain *sc; @@ -683,7 +699,7 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri) MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX)); if (pri != 0 && td->td_priority > pri) sched_prio(td, pri); - setrunnable(td); + return (setrunnable(td)); } #ifdef INVARIANTS @@ -722,18 +738,19 @@ sleepq_init(void *mem, int size, int flags) /* * Find the highest priority thread sleeping on a wait channel and resume it. */ -void +int sleepq_signal(void *wchan, int flags, int pri, int queue) { struct sleepqueue *sq; struct thread *td, *besttd; + int wakeup_swapper; CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags); KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); MPASS((queue >= 0) && (queue < NR_SLEEPQS)); sq = sleepq_lookup(wchan); if (sq == NULL) - return; + return (0); KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), ("%s: mismatch between sleep/wakeup and cv_*", __func__)); @@ -750,35 +767,40 @@ sleepq_signal(void *wchan, int flags, int pri, int queue) } MPASS(besttd != NULL); thread_lock(besttd); - sleepq_resume_thread(sq, besttd, pri); + wakeup_swapper = sleepq_resume_thread(sq, besttd, pri); thread_unlock(besttd); + return (wakeup_swapper); } /* * Resume all threads sleeping on a specified wait channel. */ -void +int sleepq_broadcast(void *wchan, int flags, int pri, int queue) { struct sleepqueue *sq; struct thread *td; + int wakeup_swapper; CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags); KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); MPASS((queue >= 0) && (queue < NR_SLEEPQS)); sq = sleepq_lookup(wchan); if (sq == NULL) - return; + return (0); KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), ("%s: mismatch between sleep/wakeup and cv_*", __func__)); /* Resume all blocked threads on the sleep queue. */ + wakeup_swapper = 0; while (!TAILQ_EMPTY(&sq->sq_blocked[queue])) { td = TAILQ_FIRST(&sq->sq_blocked[queue]); thread_lock(td); - sleepq_resume_thread(sq, td, pri); + if (sleepq_resume_thread(sq, td, pri)) + wakeup_swapper = 1; thread_unlock(td); } + return (wakeup_swapper); } /* @@ -792,8 +814,10 @@ sleepq_timeout(void *arg) struct sleepqueue *sq; struct thread *td; void *wchan; + int wakeup_swapper; td = arg; + wakeup_swapper = 0; CTR3(KTR_PROC, "sleepq_timeout: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); @@ -809,8 +833,10 @@ sleepq_timeout(void *arg) sq = sleepq_lookup(wchan); MPASS(sq != NULL); td->td_flags |= TDF_TIMEOUT; - sleepq_resume_thread(sq, td, 0); + wakeup_swapper = sleepq_resume_thread(sq, td, 0); thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); return; } @@ -839,10 +865,12 @@ sleepq_timeout(void *arg) MPASS(TD_IS_SLEEPING(td)); td->td_flags &= ~TDF_TIMEOUT; TD_CLR_SLEEPING(td); - setrunnable(td); + wakeup_swapper = setrunnable(td); } else td->td_flags |= TDF_TIMOFAIL; thread_unlock(td); + if (wakeup_swapper) + kick_proc0(); } /* @@ -853,6 +881,7 @@ void sleepq_remove(struct thread *td, void *wchan) { struct sleepqueue *sq; + int wakeup_swapper; /* * Look up the sleep queue for this wait channel, then re-check @@ -876,16 +905,18 @@ sleepq_remove(struct thread *td, void *wchan) thread_lock(td); MPASS(sq != NULL); MPASS(td->td_wchan == wchan); - sleepq_resume_thread(sq, td, 0); + wakeup_swapper = sleepq_resume_thread(sq, td, 0); thread_unlock(td); sleepq_release(wchan); + if (wakeup_swapper) + kick_proc0(); } /* * Abort a thread as if an interrupt had occurred. Only abort * interruptible waits (unfortunately it isn't safe to abort others). */ -void +int sleepq_abort(struct thread *td, int intrval) { struct sleepqueue *sq; @@ -901,7 +932,7 @@ sleepq_abort(struct thread *td, int intrval) * timeout is scheduled anyhow. */ if (td->td_flags & TDF_TIMEOUT) - return; + return (0); CTR3(KTR_PROC, "sleepq_abort: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); @@ -913,14 +944,14 @@ sleepq_abort(struct thread *td, int intrval) * we have to do it here. */ if (!TD_IS_SLEEPING(td)) - return; + return (0); wchan = td->td_wchan; MPASS(wchan != NULL); sq = sleepq_lookup(wchan); MPASS(sq != NULL); /* Thread is asleep on sleep queue sq, so wake it up. */ - sleepq_resume_thread(sq, td, 0); + return (sleepq_resume_thread(sq, td, 0)); } #ifdef SLEEPQUEUE_PROFILING diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 654e3bf8fc1a..3779591aef39 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -808,7 +808,7 @@ void pstats_free(struct pstats *ps); int securelevel_ge(struct ucred *cr, int level); int securelevel_gt(struct ucred *cr, int level); void sessrele(struct session *); -void setrunnable(struct thread *); +int setrunnable(struct thread *); void setsugid(struct proc *p); int sigonstack(size_t sp); void sleepinit(void); diff --git a/sys/sys/sleepqueue.h b/sys/sys/sleepqueue.h index 57b2c43a6985..dfd960f0bf6e 100644 --- a/sys/sys/sleepqueue.h +++ b/sys/sys/sleepqueue.h @@ -91,17 +91,17 @@ struct thread; #define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */ void init_sleepqueues(void); -void sleepq_abort(struct thread *td, int intrval); +int sleepq_abort(struct thread *td, int intrval); void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags, int queue); struct sleepqueue *sleepq_alloc(void); -void sleepq_broadcast(void *wchan, int flags, int pri, int queue); +int sleepq_broadcast(void *wchan, int flags, int pri, int queue); void sleepq_free(struct sleepqueue *sq); void sleepq_lock(void *wchan); struct sleepqueue *sleepq_lookup(void *wchan); void sleepq_release(void *wchan); void sleepq_remove(struct thread *td, void *wchan); -void sleepq_signal(void *wchan, int flags, int pri, int queue); +int sleepq_signal(void *wchan, int flags, int pri, int queue); void sleepq_set_timeout(void *wchan, int timo); int sleepq_timedwait(void *wchan, int pri); int sleepq_timedwait_sig(void *wchan, int pri); diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c index 462c4603efa1..d6ec2bad674d 100644 --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -116,10 +116,6 @@ static int swapout(struct proc *); static void swapclear(struct proc *); #endif - -static volatile int proc0_rescan; - - /* * MPSAFE * @@ -683,9 +679,6 @@ scheduler(dummy) loop: if (vm_page_count_min()) { VM_WAIT; - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } @@ -732,13 +725,7 @@ scheduler(dummy) * Nothing to do, back to sleep. */ if ((p = pp) == NULL) { - thread_lock(&thread0); - if (!proc0_rescan) { - TD_SET_IWAIT(&thread0); - mi_switch(SW_VOL | SWT_IWAIT, NULL); - } - proc0_rescan = 0; - thread_unlock(&thread0); + tsleep(&proc0, PVM, "sched", maxslp * hz / 2); goto loop; } PROC_LOCK(p); @@ -750,9 +737,6 @@ scheduler(dummy) */ if (p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) { PROC_UNLOCK(p); - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } @@ -762,32 +746,16 @@ scheduler(dummy) */ faultin(p); PROC_UNLOCK(p); - thread_lock(&thread0); - proc0_rescan = 0; - thread_unlock(&thread0); goto loop; } -void kick_proc0(void) +void +kick_proc0(void) { - struct thread *td = &thread0; - /* XXX This will probably cause a LOR in some cases */ - thread_lock(td); - if (TD_AWAITING_INTR(td)) { - CTR2(KTR_INTR, "%s: sched_add %d", __func__, 0); - TD_CLR_IWAIT(td); - sched_add(td, SRQ_INTR); - } else { - proc0_rescan = 1; - CTR2(KTR_INTR, "%s: state %d", - __func__, td->td_state); - } - thread_unlock(td); - + wakeup(&proc0); } - #ifndef NO_SWAPPING /* @@ -980,7 +948,16 @@ swapclear(p) td->td_flags &= ~TDF_SWAPINREQ; TD_CLR_SWAPPED(td); if (TD_CAN_RUN(td)) - setrunnable(td); + if (setrunnable(td)) { +#ifdef INVARIANTS + /* + * XXX: We just cleared TDI_SWAPPED + * above and set TDF_INMEM, so this + * should never happen. + */ + panic("not waking up swapper"); +#endif + } thread_unlock(td); } p->p_flag &= ~(P_SWAPPINGIN|P_SWAPPINGOUT);