From ce142d9ec03272d3f8e183ce69788ff97be0879a Mon Sep 17 00:00:00 2001 From: Alan Cox Date: Mon, 5 Jun 2006 20:35:27 +0000 Subject: [PATCH] Introduce the function pmap_enter_object(). It maps a sequence of resident pages from the same object. Use it in vm_map_pmap_enter() to reduce the locking overhead of premapping objects. Reviewed by: tegge@ --- sys/amd64/amd64/pmap.c | 52 +++++++++++++++++++++--- sys/arm/arm/pmap.c | 29 +++++++++++++- sys/i386/i386/pmap.c | 52 +++++++++++++++++++++--- sys/ia64/ia64/pmap.c | 48 ++++++++++++++++++++-- sys/powerpc/aim/mmu_oea.c | 75 ++++++++++++++++++++++++++++------- sys/powerpc/powerpc/mmu_if.m | 19 +++++++++ sys/powerpc/powerpc/mmu_oea.c | 75 ++++++++++++++++++++++++++++------- sys/sparc64/sparc64/pmap.c | 75 ++++++++++++++++++++++++++++++----- sys/vm/pmap.h | 2 + sys/vm/vm_map.c | 20 +++++++--- 10 files changed, 386 insertions(+), 61 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 8c8267ac72b1..7c0e1021dc31 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -209,6 +209,8 @@ static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try); static void pmap_clear_ptes(vm_page_t m, long bit); +static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, + vm_page_t m, vm_prot_t prot, vm_page_t mpte); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, pd_entry_t ptepde); static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde); @@ -2309,6 +2311,37 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, PMAP_UNLOCK(pmap); } +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m, mpte; + vm_pindex_t diff, psize; + + psize = atop(end - start); + mpte = NULL; + m = m_start; + PMAP_LOCK(pmap); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, + prot, mpte); + m = TAILQ_NEXT(m, listq); + } + PMAP_UNLOCK(pmap); +} + /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. @@ -2322,15 +2355,26 @@ vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { + + PMAP_LOCK(pmap); + mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte); + PMAP_UNLOCK(pmap); + return (mpte); +} + +static vm_page_t +pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, + vm_prot_t prot, vm_page_t mpte) +{ pt_entry_t *pte; vm_paddr_t pa; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, - ("pmap_enter_quick: managed mapping within the clean submap")); + ("pmap_enter_quick_locked: managed mapping within the clean submap")); mtx_assert(&vm_page_queue_mtx, MA_OWNED); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); - PMAP_LOCK(pmap); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * In the case that a page table page is not @@ -2395,7 +2439,7 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, pmap_unwire_pte_hold(pmap, va, mpte); mpte = NULL; } - goto out; + return (mpte); } /* @@ -2422,8 +2466,6 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, pte_store(pte, pa | PG_V | PG_U); else pte_store(pte, pa | PG_V | PG_U | PG_MANAGED); -out: - PMAP_UNLOCK(pmap); return mpte; } diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index 45adf5167a86..25c66cd7402d 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -3578,6 +3578,34 @@ pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, } } +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m; + vm_pindex_t diff, psize; + + psize = atop(end - start); + m = m_start; + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + pmap_enter_locked(pmap, start + ptoa(diff), m, prot & + (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); + m = TAILQ_NEXT(m, listq); + } +} + /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. @@ -3592,7 +3620,6 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { - mtx_assert(&vm_page_queue_mtx, MA_OWNED); pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); return (NULL); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index e073c1a05b17..dc0f657c2d9e 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -268,6 +268,8 @@ static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try); static void pmap_clear_ptes(vm_page_t m, int bit); +static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, + vm_page_t m, vm_prot_t prot, vm_page_t mpte); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva); static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, @@ -2385,6 +2387,37 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, PMAP_UNLOCK(pmap); } +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m, mpte; + vm_pindex_t diff, psize; + + psize = atop(end - start); + mpte = NULL; + m = m_start; + PMAP_LOCK(pmap); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, + prot, mpte); + m = TAILQ_NEXT(m, listq); + } + PMAP_UNLOCK(pmap); +} + /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. @@ -2398,15 +2431,26 @@ vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { + + PMAP_LOCK(pmap); + mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte); + PMAP_UNLOCK(pmap); + return (mpte); +} + +static vm_page_t +pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, + vm_prot_t prot, vm_page_t mpte) +{ pt_entry_t *pte; vm_paddr_t pa; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, - ("pmap_enter_quick: managed mapping within the clean submap")); + ("pmap_enter_quick_locked: managed mapping within the clean submap")); mtx_assert(&vm_page_queue_mtx, MA_OWNED); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); - PMAP_LOCK(pmap); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * In the case that a page table page is not @@ -2471,7 +2515,7 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, pmap_unwire_pte_hold(pmap, mpte); mpte = NULL; } - goto out; + return (mpte); } /* @@ -2496,8 +2540,6 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, pte_store(pte, pa | PG_V | PG_U); else pte_store(pte, pa | PG_V | PG_U | PG_MANAGED); -out: - PMAP_UNLOCK(pmap); return mpte; } diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 06d07f55a59b..0fa0d0e856bd 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -236,6 +236,8 @@ static struct ia64_lpte *pmap_find_vhpt(vm_offset_t va); static PMAP_INLINE void free_pv_entry(pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t locked_pmap); +static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, + vm_page_t m, vm_prot_t prot); static pmap_t pmap_install(pmap_t); static void pmap_invalidate_all(pmap_t pmap); static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, @@ -1625,6 +1627,35 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, PMAP_UNLOCK(pmap); } +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m; + vm_pindex_t diff, psize; + + psize = atop(end - start); + m = m_start; + PMAP_LOCK(pmap); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot); + m = TAILQ_NEXT(m, listq); + } + PMAP_UNLOCK(pmap); +} + /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. @@ -1638,16 +1669,27 @@ vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { + + PMAP_LOCK(pmap); + pmap_enter_quick_locked(pmap, va, m, prot); + PMAP_UNLOCK(pmap); + return (NULL); +} + +static void +pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, + vm_prot_t prot) +{ struct ia64_lpte *pte; pmap_t oldpmap; boolean_t managed; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, - ("pmap_enter_quick: managed mapping within the clean submap")); + ("pmap_enter_quick_locked: managed mapping within the clean submap")); mtx_assert(&vm_page_queue_mtx, MA_OWNED); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); - PMAP_LOCK(pmap); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldpmap = pmap_install(pmap); while ((pte = pmap_find_pte(va)) == NULL) { @@ -1683,8 +1725,6 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, } pmap_install(oldpmap); - PMAP_UNLOCK(pmap); - return (NULL); } /* diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 5d912716b4e6..bed7ae0448bc 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -290,6 +290,8 @@ static struct pte *moea_pvo_to_pte(const struct pvo_entry *, int); /* * Utility routines. */ +static void moea_enter_locked(pmap_t, vm_offset_t, vm_page_t, + vm_prot_t, boolean_t); static struct pvo_entry *moea_rkva_alloc(mmu_t); static void moea_pa_map(struct pvo_entry *, vm_offset_t, struct pte *, int *); @@ -309,6 +311,8 @@ void moea_clear_modify(mmu_t, vm_page_t); void moea_clear_reference(mmu_t, vm_page_t); void moea_copy_page(mmu_t, vm_page_t, vm_page_t); void moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); +void moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, + vm_prot_t); vm_page_t moea_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, vm_page_t); vm_paddr_t moea_extract(mmu_t, pmap_t, vm_offset_t); @@ -345,6 +349,7 @@ static mmu_method_t moea_methods[] = { MMUMETHOD(mmu_clear_reference, moea_clear_reference), MMUMETHOD(mmu_copy_page, moea_copy_page), MMUMETHOD(mmu_enter, moea_enter), + MMUMETHOD(mmu_enter_object, moea_enter_object), MMUMETHOD(mmu_enter_quick, moea_enter_quick), MMUMETHOD(mmu_extract, moea_extract), MMUMETHOD(mmu_extract_and_hold, moea_extract_and_hold), @@ -828,7 +833,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) struct vm_page m; m.phys_addr = translations[i].om_pa + off; - moea_enter(mmup, &ofw_pmap, + moea_enter_locked(&ofw_pmap, translations[i].om_va + off, &m, VM_PROT_ALL, 1); ofw_mappings++; @@ -1031,6 +1036,25 @@ void moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { + + vm_page_lock_queues(); + PMAP_LOCK(pmap); + pmap_enter_locked(pmap, va, m, prot, wired); + vm_page_unlock_queues(); + PMAP_UNLOCK(pmap); +} + +/* + * Map the given physical page at the specified virtual address in the + * target pmap with the protection requested. If specified the page + * will be wired down. + * + * The page queues and pmap must be locked. + */ +static void +moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, + boolean_t wired) +{ struct pvo_head *pvo_head; uma_zone_t zone; vm_page_t pg; @@ -1051,8 +1075,8 @@ moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, was_exec = 0; } if (pmap_bootstrapped) - vm_page_lock_queues(); - PMAP_LOCK(pmap); + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* XXX change the pvo head for fake pages */ if ((m->flags & PG_FICTITIOUS) == PG_FICTITIOUS) @@ -1115,12 +1139,39 @@ moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, if (pg != NULL) moea_attr_save(pg, PTE_EXEC); } - if (pmap_bootstrapped) - vm_page_unlock_queues(); /* XXX syncicache always until problems are sorted */ moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); - PMAP_UNLOCK(pmap); +} + +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +moea_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m; + vm_pindex_t diff, psize; + + psize = atop(end - start); + m = m_start; + PMAP_LOCK(pm); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + moea_enter_locked(pm, start + ptoa(diff), m, prot & + (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); + m = TAILQ_NEXT(m, listq); + } + PMAP_UNLOCK(pm); } vm_page_t @@ -1128,16 +1179,10 @@ moea_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { - vm_page_busy(m); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(m->object); - mtx_lock(&Giant); - moea_enter(mmu, pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), + PMAP_LOCK(pm); + moea_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); - mtx_unlock(&Giant); - VM_OBJECT_LOCK(m->object); - vm_page_lock_queues(); - vm_page_wakeup(m); + PMAP_UNLOCK(pm); return (NULL); } diff --git a/sys/powerpc/powerpc/mmu_if.m b/sys/powerpc/powerpc/mmu_if.m index fde7765770b0..cd9a1d589823 100644 --- a/sys/powerpc/powerpc/mmu_if.m +++ b/sys/powerpc/powerpc/mmu_if.m @@ -200,6 +200,25 @@ METHOD void enter { }; +/** + * @brief Maps a sequence of resident pages belonging to the same object. + * + * @param _pmap physical map + * @param _start virtual range start + * @param _end virtual range end + * @param _m_start physical page mapped at start + * @param _prot mapping page protection + */ +METHOD void enter_object { + mmu_t _mmu; + pmap_t _pmap; + vm_offset_t _start; + vm_offset_t _end; + vm_page_t _m_start; + vm_prot_t _prot; +}; + + /** * @brief A faster entry point for page mapping where it is possible * to short-circuit some of the tests in pmap_enter. diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c index 5d912716b4e6..bed7ae0448bc 100644 --- a/sys/powerpc/powerpc/mmu_oea.c +++ b/sys/powerpc/powerpc/mmu_oea.c @@ -290,6 +290,8 @@ static struct pte *moea_pvo_to_pte(const struct pvo_entry *, int); /* * Utility routines. */ +static void moea_enter_locked(pmap_t, vm_offset_t, vm_page_t, + vm_prot_t, boolean_t); static struct pvo_entry *moea_rkva_alloc(mmu_t); static void moea_pa_map(struct pvo_entry *, vm_offset_t, struct pte *, int *); @@ -309,6 +311,8 @@ void moea_clear_modify(mmu_t, vm_page_t); void moea_clear_reference(mmu_t, vm_page_t); void moea_copy_page(mmu_t, vm_page_t, vm_page_t); void moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); +void moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, + vm_prot_t); vm_page_t moea_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, vm_page_t); vm_paddr_t moea_extract(mmu_t, pmap_t, vm_offset_t); @@ -345,6 +349,7 @@ static mmu_method_t moea_methods[] = { MMUMETHOD(mmu_clear_reference, moea_clear_reference), MMUMETHOD(mmu_copy_page, moea_copy_page), MMUMETHOD(mmu_enter, moea_enter), + MMUMETHOD(mmu_enter_object, moea_enter_object), MMUMETHOD(mmu_enter_quick, moea_enter_quick), MMUMETHOD(mmu_extract, moea_extract), MMUMETHOD(mmu_extract_and_hold, moea_extract_and_hold), @@ -828,7 +833,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) struct vm_page m; m.phys_addr = translations[i].om_pa + off; - moea_enter(mmup, &ofw_pmap, + moea_enter_locked(&ofw_pmap, translations[i].om_va + off, &m, VM_PROT_ALL, 1); ofw_mappings++; @@ -1031,6 +1036,25 @@ void moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { + + vm_page_lock_queues(); + PMAP_LOCK(pmap); + pmap_enter_locked(pmap, va, m, prot, wired); + vm_page_unlock_queues(); + PMAP_UNLOCK(pmap); +} + +/* + * Map the given physical page at the specified virtual address in the + * target pmap with the protection requested. If specified the page + * will be wired down. + * + * The page queues and pmap must be locked. + */ +static void +moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, + boolean_t wired) +{ struct pvo_head *pvo_head; uma_zone_t zone; vm_page_t pg; @@ -1051,8 +1075,8 @@ moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, was_exec = 0; } if (pmap_bootstrapped) - vm_page_lock_queues(); - PMAP_LOCK(pmap); + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* XXX change the pvo head for fake pages */ if ((m->flags & PG_FICTITIOUS) == PG_FICTITIOUS) @@ -1115,12 +1139,39 @@ moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, if (pg != NULL) moea_attr_save(pg, PTE_EXEC); } - if (pmap_bootstrapped) - vm_page_unlock_queues(); /* XXX syncicache always until problems are sorted */ moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); - PMAP_UNLOCK(pmap); +} + +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +moea_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m; + vm_pindex_t diff, psize; + + psize = atop(end - start); + m = m_start; + PMAP_LOCK(pm); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + moea_enter_locked(pm, start + ptoa(diff), m, prot & + (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); + m = TAILQ_NEXT(m, listq); + } + PMAP_UNLOCK(pm); } vm_page_t @@ -1128,16 +1179,10 @@ moea_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { - vm_page_busy(m); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(m->object); - mtx_lock(&Giant); - moea_enter(mmu, pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), + PMAP_LOCK(pm); + moea_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); - mtx_unlock(&Giant); - VM_OBJECT_LOCK(m->object); - vm_page_lock_queues(); - vm_page_wakeup(m); + PMAP_UNLOCK(pm); return (NULL); } diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 4bf35b29f08c..43dfbd5991e6 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -147,6 +147,16 @@ struct pmap kernel_pmap_store; */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); +/* + * Map the given physical page at the specified virtual address in the + * target pmap with the protection requested. If specified the page + * will be wired down. + * + * The page queues and pmap must be locked. + */ +static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, + vm_prot_t prot, boolean_t wired); + extern int tl1_immu_miss_patch_1[]; extern int tl1_immu_miss_patch_2[]; extern int tl1_dmmu_miss_patch_1[]; @@ -1259,11 +1269,32 @@ void pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { + + vm_page_lock_queues(); + PMAP_LOCK(pm); + pmap_enter_locked(pm, va, m, prot, wired); + vm_page_unlock_queues(); + PMAP_UNLOCK(pm); +} + +/* + * Map the given physical page at the specified virtual address in the + * target pmap with the protection requested. If specified the page + * will be wired down. + * + * The page queues and pmap must be locked. + */ +static void +pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, + boolean_t wired) +{ struct tte *tp; vm_paddr_t pa; u_long data; int i; + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + PMAP_LOCK_ASSERT(pm, MA_OWNED); PMAP_STATS_INC(pmap_nenter); pa = VM_PAGE_TO_PHYS(m); @@ -1284,9 +1315,6 @@ pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); - vm_page_lock_queues(); - PMAP_LOCK(pm); - /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. @@ -1383,7 +1411,35 @@ pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, tsb_tte_enter(pm, m, va, TS_8K, data); } - vm_page_unlock_queues(); +} + +/* + * Maps a sequence of resident pages belonging to the same object. + * The sequence begins with the given page m_start. This page is + * mapped at the given virtual address start. Each subsequent page is + * mapped at a virtual address that is offset from start by the same + * amount as the page is offset from m_start within the object. The + * last page in the sequence is the page with the largest offset from + * m_start that can be mapped at a virtual address less than the given + * virtual address end. Not every virtual page between start and end + * is mapped; only those for which a resident page exists with the + * corresponding offset from m_start are mapped. + */ +void +pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, + vm_page_t m_start, vm_prot_t prot) +{ + vm_page_t m; + vm_pindex_t diff, psize; + + psize = atop(end - start); + m = m_start; + PMAP_LOCK(pm); + while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { + pmap_enter_locked(pm, start + ptoa(diff), m, prot & + (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); + m = TAILQ_NEXT(m, listq); + } PMAP_UNLOCK(pm); } @@ -1392,13 +1448,10 @@ pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { - vm_page_busy(m); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(m->object); - pmap_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); - VM_OBJECT_LOCK(m->object); - vm_page_lock_queues(); - vm_page_wakeup(m); + PMAP_LOCK(pm); + pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), + FALSE); + PMAP_UNLOCK(pm); return (NULL); } diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index 2bddd7b1655a..937fc7145ca2 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -99,6 +99,8 @@ void pmap_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte); +void pmap_enter_object(pmap_t pmap, vm_offset_t start, + vm_offset_t end, vm_page_t m_start, vm_prot_t prot); vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va); vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot); diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 93a98636f173..5fa567333510 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1437,9 +1437,9 @@ void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags) { - vm_offset_t tmpidx; + vm_offset_t start, tmpidx; int psize; - vm_page_t p, mpte; + vm_page_t p, p_start; boolean_t are_queues_locked; if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL) @@ -1465,7 +1465,8 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, } are_queues_locked = FALSE; - mpte = NULL; + start = 0; + p_start = NULL; if ((p = TAILQ_FIRST(&object->memq)) != NULL) { if (p->pindex < pindex) { @@ -1493,16 +1494,25 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { + if (p_start == NULL) { + start = addr + ptoa(tmpidx); + p_start = p; + } if (!are_queues_locked) { are_queues_locked = TRUE; vm_page_lock_queues(); } if (VM_PAGE_INQUEUE1(p, PQ_CACHE)) vm_page_deactivate(p); - mpte = pmap_enter_quick(map->pmap, - addr + ptoa(tmpidx), p, prot, mpte); + } else if (p_start != NULL) { + pmap_enter_object(map->pmap, start, addr + + ptoa(tmpidx), p_start, prot); + p_start = NULL; } } + if (p_start != NULL) + pmap_enter_object(map->pmap, start, addr + size, p_start, + prot); if (are_queues_locked) vm_page_unlock_queues(); unlock_return: