mirror of
https://git.FreeBSD.org/src.git
synced 2024-12-04 09:09:56 +00:00
Split the SLB mirror cache into two kinds of object, one for kernel maps
which are similar to the previous ones, and one for user maps, which are arrays of pointers into the SLB tree. This changes makes user SLB updates atomic, closing a window for memory corruption. While here, rearrange the allocation functions to make context switches faster.
This commit is contained in:
Nathan Whitehorn
parent
c59528330a
commit
6416b9a85d
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=212722
@ -838,7 +838,7 @@ moea64_bootstrap_slb_prefault(vm_offset_t va, int large)
|
||||
if (large)
|
||||
entry.slbv |= SLBV_L;
|
||||
|
||||
slb_insert(kernel_pmap, cache, &entry);
|
||||
slb_insert_kernel(entry.slbe, entry.slbv);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -2099,6 +2099,7 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
|
||||
|
||||
pmap->pm_slb_tree_root = slb_alloc_tree();
|
||||
pmap->pm_slb = slb_alloc_user_cache();
|
||||
pmap->pm_slb_len = 0;
|
||||
}
|
||||
#else
|
||||
void
|
||||
|
||||
@ -263,13 +263,14 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
|
||||
entry = user_va_to_slb_entry(pm, va);
|
||||
|
||||
if (entry == NULL)
|
||||
return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0));
|
||||
return (allocate_user_vsid(pm,
|
||||
(uintptr_t)va >> ADDR_SR_SHFT, 0));
|
||||
|
||||
return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
allocate_vsid(pmap_t pm, uint64_t esid, int large)
|
||||
allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
|
||||
{
|
||||
uint64_t vsid, slbv;
|
||||
struct slbtnode *ua, *next, *inter;
|
||||
@ -327,7 +328,7 @@ allocate_vsid(pmap_t pm, uint64_t esid, int large)
|
||||
* SLB mapping, so pre-spill this entry.
|
||||
*/
|
||||
eieio();
|
||||
slb_insert(pm, pm->pm_slb, slb);
|
||||
slb_insert_user(pm, slb);
|
||||
|
||||
return (vsid);
|
||||
}
|
||||
@ -410,57 +411,68 @@ slb_alloc_tree(void)
|
||||
(slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \
|
||||
(slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS)
|
||||
void
|
||||
slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry)
|
||||
slb_insert_kernel(uint64_t slbe, uint64_t slbv)
|
||||
{
|
||||
uint64_t slbe, slbv;
|
||||
int i, j, to_spill;
|
||||
struct slb *slbcache;
|
||||
int i, j;
|
||||
|
||||
/* We don't want to be preempted while modifying the kernel map */
|
||||
critical_enter();
|
||||
|
||||
to_spill = -1;
|
||||
slbv = slb_entry->slbv;
|
||||
slbe = slb_entry->slbe;
|
||||
slbcache = PCPU_GET(slb);
|
||||
|
||||
/* Hunt for a likely candidate */
|
||||
|
||||
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
|
||||
if (pm == kernel_pmap && i == USER_SR)
|
||||
continue;
|
||||
|
||||
if (!(slbcache[i].slbe & SLBE_VALID)) {
|
||||
to_spill = i;
|
||||
break;
|
||||
/* Check for an unused slot, abusing the USER_SR slot as a full flag */
|
||||
if (slbcache[USER_SR].slbe == 0) {
|
||||
for (i = 0; i < USER_SR; i++) {
|
||||
if (!(slbcache[i].slbe & SLBE_VALID))
|
||||
goto fillkernslb;
|
||||
}
|
||||
|
||||
if (to_spill < 0 && (pm != kernel_pmap ||
|
||||
SLB_SPILLABLE(slbcache[i].slbe)))
|
||||
to_spill = i;
|
||||
if (i == USER_SR)
|
||||
slbcache[USER_SR].slbe = 1;
|
||||
}
|
||||
|
||||
if (to_spill < 0)
|
||||
panic("SLB spill on ESID %#lx, but no available candidates!\n",
|
||||
(slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT);
|
||||
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
|
||||
if (i == USER_SR)
|
||||
continue;
|
||||
|
||||
if (slbcache[to_spill].slbe & SLBE_VALID) {
|
||||
/* Invalidate this first to avoid races */
|
||||
slbcache[to_spill].slbe = 0;
|
||||
mb();
|
||||
if (SLB_SPILLABLE(slbcache[i].slbe))
|
||||
break;
|
||||
}
|
||||
slbcache[to_spill].slbv = slbv;
|
||||
slbcache[to_spill].slbe = slbe | (uint64_t)to_spill;
|
||||
|
||||
KASSERT(j < 64, ("All kernel SLB slots locked!"));
|
||||
|
||||
fillkernslb:
|
||||
slbcache[i].slbv = slbv;
|
||||
slbcache[i].slbe = slbe | (uint64_t)i;
|
||||
|
||||
/* If it is for this CPU, put it in the SLB right away */
|
||||
if (pm == kernel_pmap && pmap_bootstrapped) {
|
||||
if (pmap_bootstrapped) {
|
||||
/* slbie not required */
|
||||
__asm __volatile ("slbmte %0, %1" ::
|
||||
"r"(slbcache[to_spill].slbv),
|
||||
"r"(slbcache[to_spill].slbe));
|
||||
"r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
|
||||
}
|
||||
|
||||
critical_exit();
|
||||
}
|
||||
|
||||
void
|
||||
slb_insert_user(pmap_t pm, struct slb *slb)
|
||||
{
|
||||
int i;
|
||||
|
||||
PMAP_LOCK_ASSERT(pm, MA_OWNED);
|
||||
|
||||
if (pm->pm_slb_len < 64) {
|
||||
i = pm->pm_slb_len;
|
||||
pm->pm_slb_len++;
|
||||
} else {
|
||||
i = mftb() % 64;
|
||||
}
|
||||
|
||||
/* Note that this replacement is atomic with respect to trap_subr */
|
||||
pm->pm_slb[i] = slb;
|
||||
}
|
||||
|
||||
static void
|
||||
slb_zone_init(void *dummy)
|
||||
@ -468,18 +480,18 @@ slb_zone_init(void *dummy)
|
||||
|
||||
slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
|
||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
|
||||
slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb),
|
||||
slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb *),
|
||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
|
||||
}
|
||||
|
||||
struct slb *
|
||||
struct slb **
|
||||
slb_alloc_user_cache(void)
|
||||
{
|
||||
return (uma_zalloc(slb_cache_zone, M_ZERO));
|
||||
}
|
||||
|
||||
void
|
||||
slb_free_user_cache(struct slb *slb)
|
||||
slb_free_user_cache(struct slb **slb)
|
||||
{
|
||||
uma_zfree(slb_cache_zone, slb);
|
||||
}
|
||||
|
||||
@ -445,17 +445,15 @@ syscall(struct trapframe *frame)
|
||||
static int
|
||||
handle_slb_spill(pmap_t pm, vm_offset_t addr)
|
||||
{
|
||||
struct slb kern_entry, *user_entry;
|
||||
struct slb *user_entry;
|
||||
uint64_t esid;
|
||||
int i;
|
||||
|
||||
esid = (uintptr_t)addr >> ADDR_SR_SHFT;
|
||||
|
||||
if (pm == kernel_pmap) {
|
||||
kern_entry.slbv = kernel_va_to_slbv(addr);
|
||||
kern_entry.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
|
||||
|
||||
slb_insert(pm, PCPU_GET(slb), &kern_entry);
|
||||
slb_insert_kernel((esid << SLBE_ESID_SHIFT) | SLBE_VALID,
|
||||
kernel_va_to_slbv(addr));
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -464,18 +462,18 @@ handle_slb_spill(pmap_t pm, vm_offset_t addr)
|
||||
|
||||
if (user_entry == NULL) {
|
||||
/* allocate_vsid auto-spills it */
|
||||
(void)allocate_vsid(pm, esid, 0);
|
||||
(void)allocate_user_vsid(pm, esid, 0);
|
||||
} else {
|
||||
/*
|
||||
* Check that another CPU has not already mapped this.
|
||||
* XXX: Per-thread SLB caches would be better.
|
||||
*/
|
||||
for (i = 0; i < 64; i++)
|
||||
if (pm->pm_slb[i].slbe == (user_entry->slbe | i))
|
||||
for (i = 0; i < pm->pm_slb_len; i++)
|
||||
if (pm->pm_slb[i] == user_entry)
|
||||
break;
|
||||
|
||||
if (i == 64)
|
||||
slb_insert(pm, pm->pm_slb, user_entry);
|
||||
if (i == pm->pm_slb_len)
|
||||
slb_insert_user(pm, user_entry);
|
||||
}
|
||||
PMAP_UNLOCK(pm);
|
||||
|
||||
|
||||
@ -49,45 +49,59 @@
|
||||
* Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
|
||||
*/
|
||||
|
||||
restoresrs:
|
||||
/*
|
||||
* User SRs are loaded through a pointer to the current pmap.
|
||||
*/
|
||||
restore_usersrs:
|
||||
GET_CPUINFO(%r28);
|
||||
ld %r28,PC_USERSLB(%r28);
|
||||
li %r29, 0 /* Set the counter to zero */
|
||||
|
||||
slbia
|
||||
slbmfee %r31,%r29
|
||||
clrrdi %r31,%r31,28
|
||||
slbie %r31
|
||||
instslb:
|
||||
ld %r31, 8(%r28); /* Load SLBE */
|
||||
instuserslb:
|
||||
ld %r31, 0(%r28); /* Load SLB entry pointer */
|
||||
cmpli 0, %r31, 0; /* If NULL, stop */
|
||||
beqlr;
|
||||
|
||||
cmpli 0, %r31, 0; /* If SLBE is not valid, get the next */
|
||||
beq nslb;
|
||||
|
||||
ld %r30, 0(%r28) /* Load SLBV */
|
||||
ld %r30, 0(%r31) /* Load SLBV */
|
||||
ld %r31, 8(%r31) /* Load SLBE */
|
||||
or %r31, %r31, %r29 /* Set SLBE slot */
|
||||
slbmte %r30, %r31; /* Install SLB entry */
|
||||
|
||||
nslb:
|
||||
addi %r28, %r28, 16; /* Advance */
|
||||
addi %r28, %r28, 8; /* Advance pointer */
|
||||
addi %r29, %r29, 1;
|
||||
cmpli 0, %r29, 64; /* Repeat if we are not at the end */
|
||||
blt instslb;
|
||||
|
||||
blt instuserslb;
|
||||
blr;
|
||||
|
||||
/*
|
||||
* User SRs are loaded through a pointer to the current pmap.
|
||||
* Kernel SRs are loaded directly from the PCPU fields
|
||||
*/
|
||||
#define RESTORE_USER_SRS() \
|
||||
GET_CPUINFO(%r28); \
|
||||
ld %r28,PC_USERSLB(%r28); \
|
||||
bl restoresrs;
|
||||
restore_kernsrs:
|
||||
GET_CPUINFO(%r28);
|
||||
addi %r28,%r28,PC_KERNSLB;
|
||||
li %r29, 0 /* Set the counter to zero */
|
||||
|
||||
/*
|
||||
* Kernel SRs are loaded directly from kernel_pmap_
|
||||
*/
|
||||
#define RESTORE_KERN_SRS() \
|
||||
GET_CPUINFO(%r28); \
|
||||
addi %r28,%r28,PC_KERNSLB; \
|
||||
bl restoresrs;
|
||||
slbia
|
||||
slbmfee %r31,%r29
|
||||
clrrdi %r31,%r31,28
|
||||
slbie %r31
|
||||
instkernslb:
|
||||
ld %r31, 8(%r28); /* Load SLBE */
|
||||
|
||||
cmpli 0, %r31, 0; /* If SLBE is not valid, stop */
|
||||
beqlr;
|
||||
ld %r30, 0(%r28) /* Load SLBV */
|
||||
slbmte %r30, %r31; /* Install SLB entry */
|
||||
|
||||
addi %r28, %r28, 16; /* Advance pointer */
|
||||
addi %r29, %r29, 1;
|
||||
cmpli 0, %r29, USER_SR; /* Repeat if we are not at the end */
|
||||
blt instkernslb;
|
||||
blr;
|
||||
|
||||
/*
|
||||
* FRAME_SETUP assumes:
|
||||
@ -237,7 +251,7 @@ nslb:
|
||||
std %r30,(savearea+CPUSAVE_R30)(%r3); \
|
||||
std %r31,(savearea+CPUSAVE_R31)(%r3); \
|
||||
mflr %r27; /* preserve LR */ \
|
||||
RESTORE_USER_SRS(); /* uses r28-r31 */ \
|
||||
bl restore_usersrs; /* uses r28-r31 */ \
|
||||
mtlr %r27; \
|
||||
ld %r31,(savearea+CPUSAVE_R31)(%r3); \
|
||||
ld %r30,(savearea+CPUSAVE_R30)(%r3); \
|
||||
@ -432,7 +446,7 @@ realtrap:
|
||||
ld %r1,PC_CURPCB(%r1)
|
||||
mr %r27,%r28 /* Save LR, r29 */
|
||||
mtsprg2 %r29
|
||||
RESTORE_KERN_SRS() /* enable kernel mapping */
|
||||
bl restore_kernsrs /* enable kernel mapping */
|
||||
mfsprg2 %r29
|
||||
mr %r28,%r27
|
||||
ba s_trap
|
||||
@ -482,7 +496,7 @@ u_trap:
|
||||
ld %r1,PC_CURPCB(%r1)
|
||||
mr %r27,%r28 /* Save LR, r29 */
|
||||
mtsprg2 %r29
|
||||
RESTORE_KERN_SRS() /* enable kernel mapping */
|
||||
bl restore_kernsrs /* enable kernel mapping */
|
||||
mfsprg2 %r29
|
||||
mr %r28,%r27
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ struct pmap;
|
||||
|
||||
#define PCPU_MD_AIM64_FIELDS \
|
||||
struct slb pc_slb[64]; \
|
||||
struct slb *pc_userslb;
|
||||
struct slb **pc_userslb;
|
||||
|
||||
#ifdef __powerpc64__
|
||||
#define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS
|
||||
|
||||
@ -93,7 +93,8 @@ struct pmap {
|
||||
|
||||
#ifdef __powerpc64__
|
||||
struct slbtnode *pm_slb_tree_root;
|
||||
struct slb *pm_slb;
|
||||
struct slb **pm_slb;
|
||||
int pm_slb_len;
|
||||
#else
|
||||
register_t pm_sr[16];
|
||||
#endif
|
||||
@ -142,14 +143,15 @@ uint64_t va_to_vsid(pmap_t pm, vm_offset_t va);
|
||||
uint64_t kernel_va_to_slbv(vm_offset_t va);
|
||||
struct slb *user_va_to_slb_entry(pmap_t pm, vm_offset_t va);
|
||||
|
||||
uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large);
|
||||
uint64_t allocate_user_vsid(pmap_t pm, uint64_t esid, int large);
|
||||
void free_vsid(pmap_t pm, uint64_t esid, int large);
|
||||
void slb_insert(pmap_t pm, struct slb *dst, struct slb *);
|
||||
void slb_insert_user(pmap_t pm, struct slb *slb);
|
||||
void slb_insert_kernel(uint64_t slbe, uint64_t slbv);
|
||||
|
||||
struct slbtnode *slb_alloc_tree(void);
|
||||
void slb_free_tree(pmap_t pm);
|
||||
struct slb *slb_alloc_user_cache(void);
|
||||
void slb_free_user_cache(struct slb *);
|
||||
struct slb **slb_alloc_user_cache(void);
|
||||
void slb_free_user_cache(struct slb **);
|
||||
|
||||
#else
|
||||
|
||||
|
||||
@ -42,7 +42,11 @@
|
||||
#define SR_VSID_MASK 0x00ffffff /* Virtual Segment ID mask */
|
||||
|
||||
/* Kernel segment register usage */
|
||||
#ifdef __powerpc64__
|
||||
#define USER_SR 63
|
||||
#else
|
||||
#define USER_SR 12
|
||||
#endif
|
||||
#define KERNEL_SR 13
|
||||
#define KERNEL2_SR 14
|
||||
#define KERNEL3_SR 15
|
||||
|
||||
Loading…
Reference in New Issue
Block a user