mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-12-13 10:02:38 +00:00
Code Issues Releases Activity

Cache kernel stacks in UMA. This gives us NUMA support, better concurrency,

Browse Source 
and more statistics.

Reviewed by:	kib, markj
Tested by:	pho
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D20931
This commit is contained in:
Jeff Roberson 2019-08-06 23:15:34 +00:00
parent a04725cd5c
commit 0b26119b21
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=350663
5 changed files with 101 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sys/ddb/db_ps.c
View File

@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/_kstack_cache.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
@ -505,7 +504,6 @@ db_findstack_cmd(db_expr_t addr, bool have_addr, db_expr_t dummy3 __unused,
{
struct proc *p;
struct thread *td;
struct kstack_cache_entry *ks_ce;
vm_offset_t saddr;
if (have_addr)
@ -524,13 +522,4 @@ db_findstack_cmd(db_expr_t addr, bool have_addr, db_expr_t dummy3 __unused,
}
}
}
for (ks_ce = kstack_cache; ks_ce != NULL;
ks_ce = ks_ce->next_ks_entry) {
if ((vm_offset_t)ks_ce <= saddr && saddr < (vm_offset_t)ks_ce +
PAGE_SIZE * kstack_pages) {
db_printf("Cached stack %p\n", ks_ce);
return;
}
}
}

49
sys/sys/_kstack_cache.h
View File

@ -1,49 +0,0 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2009 Konstantin Belousov <kib@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _SYS__KSTACK_CACHE_H
#define _SYS__KSTACK_CACHE_H
struct kstack_cache_entry {
struct vm_object *ksobj;
struct kstack_cache_entry *next_ks_entry;
};
extern struct kstack_cache_entry *kstack_cache;
#ifndef KSTACK_MAX_PAGES
#define KSTACK_MAX_PAGES 32
#endif
#endif

175
sys/vm/vm_glue.c
View File

@ -84,12 +84,12 @@ __FBSDID("$FreeBSD$");
#include <sys/vmem.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/_kstack_cache.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/unistd.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
@ -282,52 +282,39 @@ vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset_t sz)
pmap_sync_icache(map->pmap, va, sz);
}
struct kstack_cache_entry *kstack_cache;
static uma_zone_t kstack_cache;
static int kstack_cache_size = 128;
static int kstacks, kstack_domain_iter;
static struct mtx kstack_cache_mtx;
MTX_SYSINIT(kstack_cache, &kstack_cache_mtx, "kstkch", MTX_DEF);
static int kstack_domain_iter;
SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0,
"");
SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0,
"");
static int
sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS)
{
int error, newsize;
newsize = kstack_cache_size;
error = sysctl_handle_int(oidp, &newsize, 0, req);
if (error == 0 && req->newptr && newsize != kstack_cache_size)
kstack_cache_size =
uma_zone_set_maxcache(kstack_cache, newsize);
return (error);
}
SYSCTL_PROC(_vm, OID_AUTO, kstack_cache_size, CTLTYPE_INT|CTLFLAG_RW,
&kstack_cache_size, 0, sysctl_kstack_cache_size, "IU",
"Maximum number of cached kernel stacks");
/*
* Create the kernel stack (including pcb for i386) for a new thread.
* This routine directly affects the fork perf for a process and
* create performance for a thread.
*/
int
vm_thread_new(struct thread *td, int pages)
static vm_offset_t
vm_thread_stack_create(struct domainset *ds, vm_object_t *ksobjp, int pages)
{
vm_page_t ma[KSTACK_MAX_PAGES];
vm_object_t ksobj;
vm_offset_t ks;
vm_page_t ma[KSTACK_MAX_PAGES];
struct kstack_cache_entry *ks_ce;
int i;
/* Bounds check */
if (pages <= 1)
pages = kstack_pages;
else if (pages > KSTACK_MAX_PAGES)
pages = KSTACK_MAX_PAGES;
if (pages == kstack_pages && kstack_cache != NULL) {
mtx_lock(&kstack_cache_mtx);
if (kstack_cache != NULL) {
ks_ce = kstack_cache;
kstack_cache = ks_ce->next_ks_entry;
mtx_unlock(&kstack_cache_mtx);
td->td_kstack_obj = ks_ce->ksobj;
td->td_kstack = (vm_offset_t)ks_ce;
td->td_kstack_pages = kstack_pages;
return (1);
}
mtx_unlock(&kstack_cache_mtx);
}
/*
* Allocate an object for the kstack.
*/
@ -354,30 +341,17 @@ vm_thread_new(struct thread *td, int pages)
vm_object_deallocate(ksobj);
return (0);
}
/*
* Ensure that kstack objects can draw pages from any memory
* domain. Otherwise a local memory shortage can block a process
* swap-in.
*/
if (vm_ndomains > 1) {
ksobj->domain.dr_policy = DOMAINSET_RR();
ksobj->domain.dr_policy = ds;
ksobj->domain.dr_iter =
atomic_fetchadd_int(&kstack_domain_iter, 1);
}
atomic_add_int(&kstacks, 1);
if (KSTACK_GUARD_PAGES != 0) {
pmap_qremove(ks, KSTACK_GUARD_PAGES);
ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
}
td->td_kstack_obj = ksobj;
td->td_kstack = ks;
/*
* Knowing the number of pages allocated is useful when you
* want to deallocate them.
*/
td->td_kstack_pages = pages;
/*
* For the length of the stack, link in a real page of ram for each
* page of stack.
@ -389,7 +363,9 @@ vm_thread_new(struct thread *td, int pages)
ma[i]->valid = VM_PAGE_BITS_ALL;
VM_OBJECT_WUNLOCK(ksobj);
pmap_qenter(ks, ma, pages);
return (1);
*ksobjp = ksobj;
return (ks);
}
static void
@ -398,7 +374,6 @@ vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages)
vm_page_t m;
int i;
atomic_add_int(&kstacks, -1);
pmap_qremove(ks, pages);
VM_OBJECT_WLOCK(ksobj);
for (i = 0; i < pages; i++) {
@ -416,6 +391,45 @@ vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages)
(pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
}
/*
* Allocate the kernel stack for a new thread.
*/
int
vm_thread_new(struct thread *td, int pages)
{
vm_object_t ksobj;
vm_offset_t ks;
/* Bounds check */
if (pages <= 1)
pages = kstack_pages;
else if (pages > KSTACK_MAX_PAGES)
pages = KSTACK_MAX_PAGES;
ks = 0;
ksobj = NULL;
if (pages == kstack_pages && kstack_cache != NULL) {
ks = (vm_offset_t)uma_zalloc(kstack_cache, M_NOWAIT);
if (ks != 0)
ksobj = PHYS_TO_VM_PAGE(pmap_kextract(ks))->object;
}
/*
* Ensure that kstack objects can draw pages from any memory
* domain. Otherwise a local memory shortage can block a process
* swap-in.
*/
if (ks == 0)
ks = vm_thread_stack_create(DOMAINSET_PREF(PCPU_GET(domain)),
&ksobj, pages);
if (ks == 0)
return (0);
td->td_kstack_obj = ksobj;
td->td_kstack = ks;
td->td_kstack_pages = pages;
return (1);
}
/*
* Dispose of a thread's kernel stack.
*/
@ -424,7 +438,6 @@ vm_thread_dispose(struct thread *td)
{
vm_object_t ksobj;
vm_offset_t ks;
struct kstack_cache_entry *ks_ce;
int pages;
pages = td->td_kstack_pages;
@ -432,43 +445,49 @@ vm_thread_dispose(struct thread *td)
ks = td->td_kstack;
td->td_kstack = 0;
td->td_kstack_pages = 0;
if (pages == kstack_pages && kstacks <= kstack_cache_size) {
ks_ce = (struct kstack_cache_entry *)ks;
ks_ce->ksobj = ksobj;
mtx_lock(&kstack_cache_mtx);
ks_ce->next_ks_entry = kstack_cache;
kstack_cache = ks_ce;
mtx_unlock(&kstack_cache_mtx);
return;
if (pages == kstack_pages)
uma_zfree(kstack_cache, (void *)ks);
else
vm_thread_stack_dispose(ksobj, ks, pages);
}
static int
kstack_import(void *arg, void **store, int cnt, int domain, int flags)
{
vm_object_t ksobj;
int i;
for (i = 0; i < cnt; i++) {
store[i] = (void *)vm_thread_stack_create(
DOMAINSET_PREF(domain), &ksobj, kstack_pages);
if (store[i] == NULL)
break;
}
vm_thread_stack_dispose(ksobj, ks, pages);
return (i);
}
static void
vm_thread_stack_lowmem(void *nulll)
kstack_release(void *arg, void **store, int cnt)
{
struct kstack_cache_entry *ks_ce, *ks_ce1;
vm_offset_t ks;
int i;
mtx_lock(&kstack_cache_mtx);
ks_ce = kstack_cache;
kstack_cache = NULL;
mtx_unlock(&kstack_cache_mtx);
while (ks_ce != NULL) {
ks_ce1 = ks_ce;
ks_ce = ks_ce->next_ks_entry;
vm_thread_stack_dispose(ks_ce1->ksobj, (vm_offset_t)ks_ce1,
kstack_pages);
for (i = 0; i < cnt; i++) {
ks = (vm_offset_t)store[i];
vm_thread_stack_dispose(
PHYS_TO_VM_PAGE(pmap_kextract(ks))->object,
ks, kstack_pages);
}
}
static void
kstack_cache_init(void *nulll)
kstack_cache_init(void *null)
{
EVENTHANDLER_REGISTER(vm_lowmem, vm_thread_stack_lowmem, NULL,
EVENTHANDLER_PRI_ANY);
kstack_cache = uma_zcache_create("kstack_cache",
kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL,
kstack_import, kstack_release, NULL,
UMA_ZONE_NUMA|UMA_ZONE_MINBUCKET);
uma_zone_set_maxcache(kstack_cache, kstack_cache_size);
}
SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL);

4
sys/vm/vm_param.h
View File

@ -122,6 +122,10 @@ struct xswdev {
#endif /* !SMP */
#endif /* !PA_LOCK_COUNT */
#ifndef KSTACK_MAX_PAGES
#define KSTACK_MAX_PAGES 32
#endif
#ifndef ASSEMBLER
#ifdef _KERNEL
#define num_pages(x) \

1
sys/vm/vm_swapout.c
View File

@ -85,7 +85,6 @@ __FBSDID("$FreeBSD$");
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/_kstack_cache.h>
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/mount.h>