mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-12-23 11:18:54 +00:00
Code Issues Releases Activity

Optimize vm_radix_lookup_ge() and vm_radix_lookup_le(). Specifically,

Browse Source 
change the way that these functions ascend the tree when the search for a
matching leaf fails at an interior node.  Rather than returning to the root
of the tree and repeating the lookup with an updated key, maintain a stack
of interior nodes that were visited during the descent and use that stack
to resume the lookup at the closest ancestor that might have a matching
descendant.

Sponsored by:	EMC / Isilon Storage Division
Reviewed by:	attilio
Tested by:	pho
This commit is contained in:
Alan Cox 2013-05-04 22:50:15 +00:00
parent 06e950328a
commit 2d4b9a6438
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=250259
1 changed files with 75 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
sys/vm/vm_radix.c
View File

@ -256,54 +256,6 @@ vm_radix_keybarr(struct vm_radix_node *rnode, vm_pindex_t idx)
return (FALSE); return (FALSE);
} }
/*
* Adjusts the idx key to the first upper level available, based on a valid
* initial level and map of available levels.
* Returns a value bigger than 0 to signal that there are not valid levels
* available.
*/
static __inline int
vm_radix_addlev(vm_pindex_t *idx, boolean_t *levels, uint16_t ilev)
{
for (; levels[ilev] == FALSE ||
vm_radix_slot(*idx, ilev) == (VM_RADIX_COUNT - 1); ilev--)
if (ilev == 0)
return (1);
/*
* The following computation cannot overflow because *idx's slot at
* ilev is less than VM_RADIX_COUNT - 1.
*/
*idx = vm_radix_trimkey(*idx, ilev);
*idx += VM_RADIX_UNITLEVEL(ilev);
return (0);
}
/*
* Adjusts the idx key to the first lower level available, based on a valid
* initial level and map of available levels.
* Returns a value bigger than 0 to signal that there are not valid levels
* available.
*/
static __inline int
vm_radix_declev(vm_pindex_t *idx, boolean_t *levels, uint16_t ilev)
{
for (; levels[ilev] == FALSE ||
vm_radix_slot(*idx, ilev) == 0; ilev--)
if (ilev == 0)
return (1);
/*
* The following computation cannot overflow because *idx's slot at
* ilev is greater than 0.
*/
*idx = vm_radix_trimkey(*idx, ilev);
*idx -= 1;
return (0);
}
/* /*
* Internal helper for vm_radix_reclaim_allnodes(). * Internal helper for vm_radix_reclaim_allnodes().
* This function is recursive. * This function is recursive.
@ -499,15 +451,14 @@ vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index)
vm_page_t vm_page_t
vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index) vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
{ {
struct vm_radix_node *stack[VM_RADIX_LIMIT];
vm_pindex_t inc; vm_pindex_t inc;
vm_page_t m; vm_page_t m;
struct vm_radix_node *child, *rnode; struct vm_radix_node *child, *rnode;
int slot;
uint16_t difflev;
boolean_t maplevels[VM_RADIX_LIMIT + 1];
#ifdef INVARIANTS #ifdef INVARIANTS
int loops = 0; int loops = 0;
#endif #endif
int slot, tos;
rnode = vm_radix_getroot(rtree); rnode = vm_radix_getroot(rtree);
if (rnode == NULL) if (rnode == NULL)
@ -519,34 +470,45 @@ vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
else else
return (NULL); return (NULL);
} }
restart: tos = 0;
KASSERT(++loops < 1000, ("%s: too many loops", __func__));
for (difflev = 0; difflev < (VM_RADIX_LIMIT + 1); difflev++)
maplevels[difflev] = FALSE;
for (;;) { for (;;) {
maplevels[rnode->rn_clev] = TRUE;
/* /*
* If the keys differ before the current bisection node, * If the keys differ before the current bisection node,
* then the search key might rollback to the earliest * then the search key might rollback to the earliest
* available bisection node or to the smallest key * available bisection node or to the smallest key
* in the current node (if the owner is bigger than the * in the current node (if the owner is bigger than the
* search key). * search key).
* The maplevels array records any node has been seen
* at a given level. This aids the search for a valid
* bisection node.
*/ */
if (vm_radix_keybarr(rnode, index)) { if (vm_radix_keybarr(rnode, index)) {
if (index > rnode->rn_owner) { if (index > rnode->rn_owner) {
difflev = vm_radix_keydiff(index, ascend:
rnode->rn_owner); KASSERT(++loops < 1000,
if (vm_radix_addlev(&index, maplevels, ("vm_radix_lookup_ge: too many loops"));
difflev) > 0)
break; /*
rnode = vm_radix_getroot(rtree); * Pop nodes from the stack until either the
goto restart; * stack is empty or a node that could have a
* matching descendant is found.
*/
do {
if (tos == 0)
return (NULL);
rnode = stack[--tos];
} while (vm_radix_slot(index,
rnode->rn_clev) == (VM_RADIX_COUNT - 1));
/*
* The following computation cannot overflow
* because index's slot at the current level
* is less than VM_RADIX_COUNT - 1.
*/
index = vm_radix_trimkey(index,
rnode->rn_clev);
index += VM_RADIX_UNITLEVEL(rnode->rn_clev);
} else } else
index = rnode->rn_owner; index = rnode->rn_owner;
KASSERT(!vm_radix_keybarr(rnode, index),
("vm_radix_lookup_ge: keybarr failed"));
} }
slot = vm_radix_slot(index, rnode->rn_clev); slot = vm_radix_slot(index, rnode->rn_clev);
child = rnode->rn_child[slot]; child = rnode->rn_child[slot];
@ -580,18 +542,18 @@ vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
("vm_radix_lookup_ge: child is radix node")); ("vm_radix_lookup_ge: child is radix node"));
/* /*
* If a valid page or edge bigger than the search slot is * If a page or edge bigger than the search slot is not found
* found in the traversal, skip to the next higher-level key. * in the current node, ascend to the next higher-level node.
*/ */
if (rnode->rn_clev == 0 || vm_radix_addlev(&index, maplevels, goto ascend;
rnode->rn_clev - 1) > 0)
break;
rnode = vm_radix_getroot(rtree);
goto restart;
descend: descend:
KASSERT(rnode->rn_clev < VM_RADIX_LIMIT,
("vm_radix_lookup_ge: pushing leaf's parent"));
KASSERT(tos < VM_RADIX_LIMIT,
("vm_radix_lookup_ge: stack overflow"));
stack[tos++] = rnode;
rnode = child; rnode = child;
} }
return (NULL);
} }
/* /*
@ -600,15 +562,14 @@ vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
vm_page_t vm_page_t
vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index) vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index)
{ {
struct vm_radix_node *stack[VM_RADIX_LIMIT];
vm_pindex_t inc; vm_pindex_t inc;
vm_page_t m; vm_page_t m;
struct vm_radix_node *child, *rnode; struct vm_radix_node *child, *rnode;
int slot;
uint16_t difflev;
boolean_t maplevels[VM_RADIX_LIMIT + 1];
#ifdef INVARIANTS #ifdef INVARIANTS
int loops = 0; int loops = 0;
#endif #endif
int slot, tos;
rnode = vm_radix_getroot(rtree); rnode = vm_radix_getroot(rtree);
if (rnode == NULL) if (rnode == NULL)
@ -620,36 +581,47 @@ vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index)
else else
return (NULL); return (NULL);
} }
restart: tos = 0;
KASSERT(++loops < 1000, ("%s: too many loops", __func__));
for (difflev = 0; difflev < (VM_RADIX_LIMIT + 1); difflev++)
maplevels[difflev] = FALSE;
for (;;) { for (;;) {
maplevels[rnode->rn_clev] = TRUE;
/* /*
* If the keys differ before the current bisection node, * If the keys differ before the current bisection node,
* then the search key might rollback to the earliest * then the search key might rollback to the earliest
* available bisection node or to the largest key * available bisection node or to the largest key
* in the current node (if the owner is smaller than the * in the current node (if the owner is smaller than the
* search key). * search key).
* The maplevels array records any node has been seen
* at a given level. This aids the search for a valid
* bisection node.
*/ */
if (vm_radix_keybarr(rnode, index)) { if (vm_radix_keybarr(rnode, index)) {
if (index > rnode->rn_owner) { if (index > rnode->rn_owner) {
index = rnode->rn_owner + VM_RADIX_COUNT * index = rnode->rn_owner + VM_RADIX_COUNT *
VM_RADIX_UNITLEVEL(rnode->rn_clev) - 1; VM_RADIX_UNITLEVEL(rnode->rn_clev);
} else { } else {
difflev = vm_radix_keydiff(index, ascend:
rnode->rn_owner); KASSERT(++loops < 1000,
if (vm_radix_declev(&index, maplevels, ("vm_radix_lookup_le: too many loops"));
difflev) > 0)
break; /*
rnode = vm_radix_getroot(rtree); * Pop nodes from the stack until either the
goto restart; * stack is empty or a node that could have a
* matching descendant is found.
*/
do {
if (tos == 0)
return (NULL);
rnode = stack[--tos];
} while (vm_radix_slot(index,
rnode->rn_clev) == 0);
/*
* The following computation cannot overflow
* because index's slot at the current level
* is greater than 0.
*/
index = vm_radix_trimkey(index,
rnode->rn_clev);
} }
index--;
KASSERT(!vm_radix_keybarr(rnode, index),
("vm_radix_lookup_le: keybarr failed"));
} }
slot = vm_radix_slot(index, rnode->rn_clev); slot = vm_radix_slot(index, rnode->rn_clev);
child = rnode->rn_child[slot]; child = rnode->rn_child[slot];
@ -683,18 +655,18 @@ vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index)
("vm_radix_lookup_le: child is radix node")); ("vm_radix_lookup_le: child is radix node"));
/* /*
* If a valid page or edge smaller than the search slot is * If a page or edge smaller than the search slot is not found
* found in the traversal, skip to the next higher-level key. * in the current node, ascend to the next higher-level node.
*/ */
if (rnode->rn_clev == 0 || vm_radix_declev(&index, maplevels, goto ascend;
rnode->rn_clev - 1) > 0)
break;
rnode = vm_radix_getroot(rtree);
goto restart;
descend: descend:
KASSERT(rnode->rn_clev < VM_RADIX_LIMIT,
("vm_radix_lookup_le: pushing leaf's parent"));
KASSERT(tos < VM_RADIX_LIMIT,
("vm_radix_lookup_le: stack overflow"));
stack[tos++] = rnode;
rnode = child; rnode = child;
} }
return (NULL);
} }
/* /*