mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-11-21 07:15:49 +00:00
Code Issues Releases Activity

amd64: use INVLPGB for kernel pmap invalidations

Browse Source 
avoiding broadcast IPIs.

Reviewed by:	alc, markj
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D45191
This commit is contained in:
Konstantin Belousov 2024-05-14 01:42:08 +03:00
parent bc4ffcadf2
commit 47656cc1ef
1 changed files with 54 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
sys/amd64/amd64/mp_machdep.c
View File

@ -679,6 +679,20 @@ smp_targeted_tlb_shootdown_native(pmap_t pmap, vm_offset_t addr1, vm_offset_t ad
void
smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
if (invlpgb_works && pmap == kernel_pmap) {
invlpgb(INVLPGB_GLOB, 0, 0);
/*
* TLBSYNC syncs only against INVLPGB executed on the
* same CPU. Since current thread is pinned by
* caller, we do not need to enter critical section to
* prevent migration.
*/
tlbsync();
sched_unpin();
return;
}
smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
@ -688,6 +702,13 @@ smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
void
smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
if (invlpgb_works && pmap == kernel_pmap) {
invlpgb(INVLPGB_GLOB | INVLPGB_VA | trunc_page(addr), 0, 0);
tlbsync();
sched_unpin();
return;
}
smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
@ -698,6 +719,39 @@ void
smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{
if (invlpgb_works && pmap == kernel_pmap) {
vm_offset_t va;
uint64_t cnt, total;
addr1 = trunc_page(addr1);
addr2 = round_page(addr2);
total = atop(addr2 - addr1);
for (va = addr1; total > 0;) {
if ((va & PDRMASK) != 0 || total < NPDEPG) {
cnt = atop(NBPDR - (va & PDRMASK));
if (cnt > total)
cnt = total;
if (cnt > invlpgb_maxcnt + 1)
cnt = invlpgb_maxcnt + 1;
invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
cnt - 1);
va += ptoa(cnt);
total -= cnt;
} else {
cnt = total / NPTEPG;
if (cnt > invlpgb_maxcnt + 1)
cnt = invlpgb_maxcnt + 1;
invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
INVLPGB_2M_CNT | (cnt - 1));
va += cnt << PDRSHIFT;
total -= cnt * NPTEPG;
}
}
tlbsync();
sched_unpin();
return;
}
smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
invl_op_pgrng);
#ifdef COUNT_XINVLTLB_HITS