diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c index 789a93a13e7b..41a55580f336 100644 --- a/sys/alpha/alpha/pmap.c +++ b/sys/alpha/alpha/pmap.c @@ -153,6 +153,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -334,7 +335,7 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va); -static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex); +static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m); static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); #ifdef SMP @@ -1038,24 +1039,28 @@ pmap_pinit(pmap) * mapped correctly. */ static vm_page_t -_pmap_allocpte(pmap, ptepindex) - pmap_t pmap; - unsigned ptepindex; +_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) { pt_entry_t* pte; vm_offset_t ptepa; vm_page_t m; + KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || + (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, + ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); + /* * Find or fabricate a new pagetable page */ if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { - PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); - VM_WAIT; - vm_page_lock_queues(); - PMAP_LOCK(pmap); + if (flags & M_WAITOK) { + PMAP_UNLOCK(pmap); + vm_page_unlock_queues(); + VM_WAIT; + vm_page_lock_queues(); + PMAP_LOCK(pmap); + } /* * Indicate the need to retry. While waiting, the page table @@ -1082,7 +1087,8 @@ _pmap_allocpte(pmap, ptepindex) pt_entry_t* l1pte = &pmap->pm_lev1[l1index]; pt_entry_t* l2map; if (!pmap_pte_v(l1pte)) { - if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index) == NULL) { + if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index, + flags) == NULL) { --m->wire_count; vm_page_free(m); return (NULL); @@ -1146,7 +1152,7 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va) * Here if the pte page isn't mapped, or if it has been * deallocated. */ - m = _pmap_allocpte(pmap, ptepindex); + m = _pmap_allocpte(pmap, ptepindex, M_WAITOK); if (m == NULL) goto retry; } @@ -1842,9 +1848,20 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) } mpte->wire_count++; } else { - mpte = _pmap_allocpte(pmap, ptepindex); - if (mpte == NULL) + mpte = _pmap_allocpte(pmap, ptepindex, + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index a19c721c3bda..e089d62cbc03 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -2053,9 +2053,19 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) mpte->wire_count++; } else { mpte = _pmap_allocpte(pmap, ptepindex, - M_WAITOK); - if (mpte == NULL) + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index f9dca0947866..91597873c0f7 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -3417,8 +3417,19 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + mtx_lock(&Giant); pmap_enter(pmap, va, m, VM_PROT_READ|VM_PROT_EXECUTE, FALSE); pmap_dcache_wbinv_all(pmap); /* XXX: shouldn't be needed */ + mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index d16bf02123e5..67ac6f6a201c 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -2094,9 +2094,19 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) mpte->wire_count++; } else { mpte = _pmap_allocpte(pmap, ptepindex, - M_WAITOK); - if (mpte == NULL) + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index da1bcf1eccb4..a2feed7a8a4c 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -1631,9 +1631,13 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) while ((pte = pmap_find_pte(va)) == NULL) { pmap_install(oldpmap); PMAP_UNLOCK(pmap); + vm_page_busy(m); vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); VM_WAIT; + VM_OBJECT_LOCK(m->object); vm_page_lock_queues(); + vm_page_wakeup(m); PMAP_LOCK(pmap); oldpmap = pmap_install(pmap); } diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 5398d61c111f..136e412847ca 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c index 5398d61c111f..136e412847ca 100644 --- a/sys/powerpc/powerpc/mmu_oea.c +++ b/sys/powerpc/powerpc/mmu_oea.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/powerpc/powerpc/pmap.c b/sys/powerpc/powerpc/pmap.c index 5398d61c111f..136e412847ca 100644 --- a/sys/powerpc/powerpc/pmap.c +++ b/sys/powerpc/powerpc/pmap.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 18cd0f9578db..fa964ab757f3 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1401,9 +1401,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 4bd82f4c678e..6fac17cba450 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -973,23 +973,17 @@ vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry) VM_OBJECT_UNLOCK(lobject); break; } - vm_page_lock_queues(); if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((m->queue - m->pc) == PQ_CACHE) { + vm_page_lock_queues(); vm_page_deactivate(m); + vm_page_unlock_queues(); } - vm_page_busy(m); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(lobject); mpte = pmap_enter_quick(pmap, addr, m, mpte); - VM_OBJECT_LOCK(lobject); - vm_page_lock_queues(); - vm_page_wakeup(m); } - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(lobject); } } diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 3788a4f119f1..46ed8490aa47 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1417,22 +1417,17 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, cnt.v_free_count < cnt.v_free_reserved) { break; } - vm_page_lock_queues(); if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { - if ((p->queue - p->pc) == PQ_CACHE) + if ((p->queue - p->pc) == PQ_CACHE) { + vm_page_lock_queues(); vm_page_deactivate(p); - vm_page_busy(p); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(object); + vm_page_unlock_queues(); + } mpte = pmap_enter_quick(map->pmap, addr + ptoa(tmpidx), p, mpte); - VM_OBJECT_LOCK(object); - vm_page_lock_queues(); - vm_page_wakeup(p); } - vm_page_unlock_queues(); } unlock_return: VM_OBJECT_UNLOCK(object);