2005-01-06 23:35:40 +00:00
|
|
|
/*-
|
1994-05-24 10:09:53 +00:00
|
|
|
* Copyright (c) 1987, 1991, 1993
|
|
|
|
* The Regents of the University of California. All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94
|
|
|
|
*/
|
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
1998-02-23 07:41:23 +00:00
|
|
|
#include "opt_vm.h"
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/param.h>
|
1994-05-25 09:21:21 +00:00
|
|
|
#include <sys/systm.h>
|
2004-07-10 21:36:01 +00:00
|
|
|
#include <sys/kdb.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/kernel.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/lock.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/malloc.h>
|
1996-01-29 09:58:34 +00:00
|
|
|
#include <sys/mbuf.h>
|
2000-10-20 07:29:16 +00:00
|
|
|
#include <sys/mutex.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <sys/vmmeter.h>
|
2001-01-21 19:25:07 +00:00
|
|
|
#include <sys/proc.h>
|
2002-04-15 04:05:53 +00:00
|
|
|
#include <sys/sysctl.h>
|
2002-11-01 18:58:12 +00:00
|
|
|
#include <sys/time.h>
|
2000-09-23 00:01:37 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <vm/vm.h>
|
2002-09-18 08:26:30 +00:00
|
|
|
#include <vm/pmap.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <vm/vm_param.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <vm/vm_kern.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <vm/vm_extern.h>
|
1997-08-05 00:02:08 +00:00
|
|
|
#include <vm/vm_map.h>
|
2002-09-18 08:26:30 +00:00
|
|
|
#include <vm/vm_page.h>
|
2002-03-19 09:11:49 +00:00
|
|
|
#include <vm/uma.h>
|
|
|
|
#include <vm/uma_int.h>
|
2002-04-30 07:54:25 +00:00
|
|
|
#include <vm/uma_dbg.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
#include <vm/memguard.h>
|
|
|
|
#endif
|
|
|
|
|
1999-09-19 08:40:11 +00:00
|
|
|
#if defined(INVARIANTS) && defined(__i386__)
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
#endif
|
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/*
|
|
|
|
* When realloc() is called, if the new size is sufficiently smaller than
|
|
|
|
* the old size, realloc() will allocate a new, smaller block to avoid
|
|
|
|
* wasting memory. 'Sufficiently smaller' is defined as: newsize <=
|
|
|
|
* oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'.
|
|
|
|
*/
|
|
|
|
#ifndef REALLOC_FRACTION
|
|
|
|
#define REALLOC_FRACTION 1 /* new block if <= half the size */
|
|
|
|
#endif
|
|
|
|
|
1999-10-03 12:18:29 +00:00
|
|
|
MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches");
|
1999-09-11 16:41:39 +00:00
|
|
|
MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory");
|
|
|
|
MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers");
|
|
|
|
|
1999-11-22 02:45:11 +00:00
|
|
|
MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");
|
|
|
|
MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery");
|
|
|
|
|
2002-03-19 21:25:46 +00:00
|
|
|
static void kmeminit(void *);
|
1995-08-28 09:19:25 +00:00
|
|
|
SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL)
|
|
|
|
|
1997-10-12 20:26:33 +00:00
|
|
|
static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
|
|
|
|
|
1998-11-10 08:46:24 +00:00
|
|
|
static struct malloc_type *kmemstatistics;
|
1997-10-10 14:06:34 +00:00
|
|
|
static char *kmembase;
|
1997-09-16 13:52:04 +00:00
|
|
|
static char *kmemlimit;
|
2000-02-16 21:11:33 +00:00
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
#define KMEM_ZSHIFT 4
|
|
|
|
#define KMEM_ZBASE 16
|
|
|
|
#define KMEM_ZMASK (KMEM_ZBASE - 1)
|
|
|
|
|
2003-09-19 04:39:08 +00:00
|
|
|
#define KMEM_ZMAX PAGE_SIZE
|
2002-03-19 09:11:49 +00:00
|
|
|
#define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT)
|
2002-04-15 04:05:53 +00:00
|
|
|
static u_int8_t kmemsize[KMEM_ZSIZE + 1];
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
/* These won't be powers of two for long */
|
|
|
|
struct {
|
2002-04-15 04:05:53 +00:00
|
|
|
int kz_size;
|
|
|
|
char *kz_name;
|
|
|
|
uma_zone_t kz_zone;
|
|
|
|
} kmemzones[] = {
|
|
|
|
{16, "16", NULL},
|
|
|
|
{32, "32", NULL},
|
|
|
|
{64, "64", NULL},
|
|
|
|
{128, "128", NULL},
|
|
|
|
{256, "256", NULL},
|
|
|
|
{512, "512", NULL},
|
|
|
|
{1024, "1024", NULL},
|
|
|
|
{2048, "2048", NULL},
|
|
|
|
{4096, "4096", NULL},
|
2003-09-19 04:39:08 +00:00
|
|
|
#if PAGE_SIZE > 4096
|
2002-04-15 04:05:53 +00:00
|
|
|
{8192, "8192", NULL},
|
2003-09-19 04:39:08 +00:00
|
|
|
#if PAGE_SIZE > 8192
|
2002-04-29 17:53:23 +00:00
|
|
|
{16384, "16384", NULL},
|
2003-09-19 04:39:08 +00:00
|
|
|
#if PAGE_SIZE > 16384
|
2002-04-29 17:12:02 +00:00
|
|
|
{32768, "32768", NULL},
|
2003-09-19 04:39:08 +00:00
|
|
|
#if PAGE_SIZE > 32768
|
2002-04-29 17:12:02 +00:00
|
|
|
{65536, "65536", NULL},
|
2003-09-19 04:39:08 +00:00
|
|
|
#if PAGE_SIZE > 65536
|
|
|
|
#error "Unsupported PAGE_SIZE"
|
|
|
|
#endif /* 65536 */
|
|
|
|
#endif /* 32768 */
|
|
|
|
#endif /* 16384 */
|
|
|
|
#endif /* 8192 */
|
|
|
|
#endif /* 4096 */
|
2002-03-19 09:11:49 +00:00
|
|
|
{0, NULL},
|
1994-05-24 10:09:53 +00:00
|
|
|
};
|
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
u_int vm_memguard_divisor;
|
|
|
|
SYSCTL_UINT(_vm, OID_AUTO, memguard_divisor, CTLFLAG_RD, &vm_memguard_divisor,
|
|
|
|
0, "(kmem_size/memguard_divisor) == memguard submap size");
|
|
|
|
#endif
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
u_int vm_kmem_size;
|
2004-01-27 15:59:38 +00:00
|
|
|
SYSCTL_UINT(_vm, OID_AUTO, kmem_size, CTLFLAG_RD, &vm_kmem_size, 0,
|
|
|
|
"Size of kernel memory");
|
2002-05-02 07:22:19 +00:00
|
|
|
|
2004-09-29 14:21:40 +00:00
|
|
|
u_int vm_kmem_size_max;
|
|
|
|
SYSCTL_UINT(_vm, OID_AUTO, kmem_size_max, CTLFLAG_RD, &vm_kmem_size_max, 0,
|
|
|
|
"Maximum size of kernel memory");
|
|
|
|
|
|
|
|
u_int vm_kmem_size_scale;
|
|
|
|
SYSCTL_UINT(_vm, OID_AUTO, kmem_size_scale, CTLFLAG_RD, &vm_kmem_size_scale, 0,
|
|
|
|
"Scale factor for kernel memory size");
|
|
|
|
|
2002-05-02 07:22:19 +00:00
|
|
|
/*
|
2002-09-18 08:26:30 +00:00
|
|
|
* The malloc_mtx protects the kmemstatistics linked list.
|
2002-05-02 07:22:19 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
struct mtx malloc_mtx;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2002-04-15 05:24:01 +00:00
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
uint64_t krequests[KMEM_ZSIZE + 1];
|
|
|
|
|
|
|
|
static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
|
|
|
|
#endif
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-04-15 05:24:01 +00:00
|
|
|
static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2002-11-01 18:58:12 +00:00
|
|
|
/* time_uptime of last malloc(9) failure */
|
|
|
|
static time_t t_malloc_fail;
|
|
|
|
|
2003-03-26 20:18:40 +00:00
|
|
|
#ifdef MALLOC_MAKE_FAILURES
|
|
|
|
/*
|
|
|
|
* Causes malloc failures every (n) mallocs with M_NOWAIT. If set to 0,
|
|
|
|
* doesn't cause failures.
|
|
|
|
*/
|
|
|
|
SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
|
|
|
|
"Kernel malloc debugging options");
|
|
|
|
|
|
|
|
static int malloc_failure_rate;
|
|
|
|
static int malloc_nowait_count;
|
|
|
|
static int malloc_failure_count;
|
|
|
|
SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RW,
|
|
|
|
&malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail");
|
2003-03-26 20:44:29 +00:00
|
|
|
TUNABLE_INT("debug.malloc.failure_rate", &malloc_failure_rate);
|
2003-03-26 20:18:40 +00:00
|
|
|
SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
|
|
|
|
&malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
|
|
|
|
#endif
|
|
|
|
|
2002-11-01 18:58:12 +00:00
|
|
|
int
|
|
|
|
malloc_last_fail(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (time_uptime - t_malloc_fail);
|
|
|
|
}
|
|
|
|
|
Reimplement contigmalloc(9) with an algorithm which stands a greatly-
improved chance of working despite pressure from running programs.
Instead of trying to throw a bunch of pages out to swap and hope for
the best, only a range that can potentially fulfill contigmalloc(9)'s
request will have its contents paged out (potentially, not forcibly)
at a time.
The new contigmalloc operation still operates in three passes, but it
could potentially be tuned to more or less. The first pass only looks
at pages in the cache and free pages, so they would be thrown out
without having to block. If this is not enough, the subsequent passes
page out any unwired memory. To combat memory pressure refragmenting
the section of memory being laundered, each page is removed from the
systems' free memory queue once it has been freed so that blocking
later doesn't cause the memory laundered so far to get reallocated.
The page-out operations are now blocking, as it would make little sense
to try to push out a page, then get its status immediately afterward
to remove it from the available free pages queue, if it's unlikely to
have been freed. Another change is that if KVA allocation fails, the
allocated memory segment will be freed and not leaked.
There is a sysctl/tunable, defaulting to on, which causes the old
contigmalloc() algorithm to be used. Nonetheless, I have been using
vm.old_contigmalloc=0 for over a month. It is safe to switch at
run-time to see the difference it makes.
A new interface has been used which does not require mapping the
allocated pages into KVA: vm_page.h functions vm_page_alloc_contig()
and vm_page_release_contig(). These are what vm.old_contigmalloc=0
uses internally, so the sysctl/tunable does not affect their operation.
When using the contigmalloc(9) and contigfree(9) interfaces, memory
is now tracked with malloc(9) stats. Several functions have been
exported from kern_malloc.c to allow other subsystems to use these
statistics, as well. This invalidates the BUGS section of the
contigmalloc(9) manpage.
2004-07-19 06:21:27 +00:00
|
|
|
/*
|
|
|
|
* Add this to the informational malloc_type bucket.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
malloc_type_zone_allocated(struct malloc_type *ksp, unsigned long size,
|
|
|
|
int zindx)
|
|
|
|
{
|
|
|
|
mtx_lock(&ksp->ks_mtx);
|
|
|
|
ksp->ks_calls++;
|
|
|
|
if (zindx != -1)
|
|
|
|
ksp->ks_size |= 1 << zindx;
|
|
|
|
if (size != 0) {
|
|
|
|
ksp->ks_memuse += size;
|
|
|
|
ksp->ks_inuse++;
|
|
|
|
if (ksp->ks_memuse > ksp->ks_maxused)
|
|
|
|
ksp->ks_maxused = ksp->ks_memuse;
|
|
|
|
}
|
|
|
|
mtx_unlock(&ksp->ks_mtx);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
malloc_type_allocated(struct malloc_type *ksp, unsigned long size)
|
|
|
|
{
|
|
|
|
malloc_type_zone_allocated(ksp, size, -1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove this allocation from the informational malloc_type bucket.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
malloc_type_freed(struct malloc_type *ksp, unsigned long size)
|
|
|
|
{
|
|
|
|
mtx_lock(&ksp->ks_mtx);
|
|
|
|
KASSERT(size <= ksp->ks_memuse,
|
|
|
|
("malloc(9)/free(9) confusion.\n%s",
|
|
|
|
"Probably freeing with wrong type, but maybe not here."));
|
|
|
|
ksp->ks_memuse -= size;
|
|
|
|
ksp->ks_inuse--;
|
|
|
|
mtx_unlock(&ksp->ks_mtx);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
1999-01-21 08:29:12 +00:00
|
|
|
* malloc:
|
|
|
|
*
|
|
|
|
* Allocate a block of memory.
|
|
|
|
*
|
|
|
|
* If M_NOWAIT is set, this routine will not block and return NULL if
|
|
|
|
* the allocation fails.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
void *
|
2005-04-12 23:54:34 +00:00
|
|
|
malloc(unsigned long size, struct malloc_type *type, int flags)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-04-15 04:05:53 +00:00
|
|
|
int indx;
|
2002-03-19 09:11:49 +00:00
|
|
|
caddr_t va;
|
|
|
|
uma_zone_t zone;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
uma_keg_t keg;
|
2003-02-01 10:07:49 +00:00
|
|
|
#ifdef DIAGNOSTIC
|
|
|
|
unsigned long osize = size;
|
|
|
|
#endif
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2003-03-10 20:24:54 +00:00
|
|
|
#ifdef INVARIANTS
|
2003-03-10 19:39:53 +00:00
|
|
|
/*
|
|
|
|
* To make sure that WAITOK or NOWAIT is set, but not more than
|
|
|
|
* one, and check against the API botches that are common.
|
|
|
|
*/
|
|
|
|
indx = flags & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT);
|
|
|
|
if (indx != M_NOWAIT && indx != M_WAITOK) {
|
|
|
|
static struct timeval lasterr;
|
|
|
|
static int curerr, once;
|
|
|
|
if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) {
|
|
|
|
printf("Bad malloc flags: %x\n", indx);
|
2004-07-10 21:36:01 +00:00
|
|
|
kdb_backtrace();
|
2003-03-10 19:39:53 +00:00
|
|
|
flags |= M_WAITOK;
|
|
|
|
once++;
|
|
|
|
}
|
|
|
|
}
|
2003-03-10 20:24:54 +00:00
|
|
|
#endif
|
2002-04-23 18:50:25 +00:00
|
|
|
#if 0
|
|
|
|
if (size == 0)
|
2004-07-10 21:36:01 +00:00
|
|
|
kdb_enter("zero size malloc");
|
2003-03-26 20:18:40 +00:00
|
|
|
#endif
|
|
|
|
#ifdef MALLOC_MAKE_FAILURES
|
|
|
|
if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) {
|
|
|
|
atomic_add_int(&malloc_nowait_count, 1);
|
|
|
|
if ((malloc_nowait_count % malloc_failure_rate) == 0) {
|
|
|
|
atomic_add_int(&malloc_failure_count, 1);
|
2003-04-25 21:49:24 +00:00
|
|
|
t_malloc_fail = time_uptime;
|
2003-03-26 20:18:40 +00:00
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
}
|
2002-04-23 18:50:25 +00:00
|
|
|
#endif
|
2003-03-10 19:39:53 +00:00
|
|
|
if (flags & M_WAITOK)
|
2001-09-12 08:38:13 +00:00
|
|
|
KASSERT(curthread->td_intr_nesting_level == 0,
|
2003-02-19 05:47:46 +00:00
|
|
|
("malloc(M_WAITOK) in interrupt context"));
|
2005-01-21 18:09:17 +00:00
|
|
|
|
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
/* XXX CHANGEME! */
|
|
|
|
if (type == M_SUBPROC)
|
|
|
|
return memguard_alloc(size, flags);
|
|
|
|
#endif
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
if (size <= KMEM_ZMAX) {
|
2002-04-15 04:05:53 +00:00
|
|
|
if (size & KMEM_ZMASK)
|
|
|
|
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
|
|
|
|
indx = kmemsize[size >> KMEM_ZSHIFT];
|
|
|
|
zone = kmemzones[indx].kz_zone;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
keg = zone->uz_keg;
|
2002-04-15 04:05:53 +00:00
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
krequests[size >> KMEM_ZSHIFT]++;
|
|
|
|
#endif
|
2002-03-19 09:11:49 +00:00
|
|
|
va = uma_zalloc(zone, flags);
|
Reimplement contigmalloc(9) with an algorithm which stands a greatly-
improved chance of working despite pressure from running programs.
Instead of trying to throw a bunch of pages out to swap and hope for
the best, only a range that can potentially fulfill contigmalloc(9)'s
request will have its contents paged out (potentially, not forcibly)
at a time.
The new contigmalloc operation still operates in three passes, but it
could potentially be tuned to more or less. The first pass only looks
at pages in the cache and free pages, so they would be thrown out
without having to block. If this is not enough, the subsequent passes
page out any unwired memory. To combat memory pressure refragmenting
the section of memory being laundered, each page is removed from the
systems' free memory queue once it has been freed so that blocking
later doesn't cause the memory laundered so far to get reallocated.
The page-out operations are now blocking, as it would make little sense
to try to push out a page, then get its status immediately afterward
to remove it from the available free pages queue, if it's unlikely to
have been freed. Another change is that if KVA allocation fails, the
allocated memory segment will be freed and not leaked.
There is a sysctl/tunable, defaulting to on, which causes the old
contigmalloc() algorithm to be used. Nonetheless, I have been using
vm.old_contigmalloc=0 for over a month. It is safe to switch at
run-time to see the difference it makes.
A new interface has been used which does not require mapping the
allocated pages into KVA: vm_page.h functions vm_page_alloc_contig()
and vm_page_release_contig(). These are what vm.old_contigmalloc=0
uses internally, so the sysctl/tunable does not affect their operation.
When using the contigmalloc(9) and contigfree(9) interfaces, memory
is now tracked with malloc(9) stats. Several functions have been
exported from kern_malloc.c to allow other subsystems to use these
statistics, as well. This invalidates the BUGS section of the
contigmalloc(9) manpage.
2004-07-19 06:21:27 +00:00
|
|
|
if (va != NULL)
|
|
|
|
size = keg->uk_size;
|
|
|
|
malloc_type_zone_allocated(type, va == NULL ? 0 : size, indx);
|
2002-03-19 09:11:49 +00:00
|
|
|
} else {
|
2002-04-15 04:05:53 +00:00
|
|
|
size = roundup(size, PAGE_SIZE);
|
2002-03-19 09:11:49 +00:00
|
|
|
zone = NULL;
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
keg = NULL;
|
2002-03-19 09:11:49 +00:00
|
|
|
va = uma_large_malloc(size, flags);
|
Reimplement contigmalloc(9) with an algorithm which stands a greatly-
improved chance of working despite pressure from running programs.
Instead of trying to throw a bunch of pages out to swap and hope for
the best, only a range that can potentially fulfill contigmalloc(9)'s
request will have its contents paged out (potentially, not forcibly)
at a time.
The new contigmalloc operation still operates in three passes, but it
could potentially be tuned to more or less. The first pass only looks
at pages in the cache and free pages, so they would be thrown out
without having to block. If this is not enough, the subsequent passes
page out any unwired memory. To combat memory pressure refragmenting
the section of memory being laundered, each page is removed from the
systems' free memory queue once it has been freed so that blocking
later doesn't cause the memory laundered so far to get reallocated.
The page-out operations are now blocking, as it would make little sense
to try to push out a page, then get its status immediately afterward
to remove it from the available free pages queue, if it's unlikely to
have been freed. Another change is that if KVA allocation fails, the
allocated memory segment will be freed and not leaked.
There is a sysctl/tunable, defaulting to on, which causes the old
contigmalloc() algorithm to be used. Nonetheless, I have been using
vm.old_contigmalloc=0 for over a month. It is safe to switch at
run-time to see the difference it makes.
A new interface has been used which does not require mapping the
allocated pages into KVA: vm_page.h functions vm_page_alloc_contig()
and vm_page_release_contig(). These are what vm.old_contigmalloc=0
uses internally, so the sysctl/tunable does not affect their operation.
When using the contigmalloc(9) and contigfree(9) interfaces, memory
is now tracked with malloc(9) stats. Several functions have been
exported from kern_malloc.c to allow other subsystems to use these
statistics, as well. This invalidates the BUGS section of the
contigmalloc(9) manpage.
2004-07-19 06:21:27 +00:00
|
|
|
malloc_type_allocated(type, va == NULL ? 0 : size);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2003-05-12 05:09:56 +00:00
|
|
|
if (flags & M_WAITOK)
|
2003-02-19 05:47:46 +00:00
|
|
|
KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
|
2003-05-12 05:09:56 +00:00
|
|
|
else if (va == NULL)
|
2002-11-01 18:58:12 +00:00
|
|
|
t_malloc_fail = time_uptime;
|
2003-02-01 10:07:49 +00:00
|
|
|
#ifdef DIAGNOSTIC
|
2003-05-12 05:09:56 +00:00
|
|
|
if (va != NULL && !(flags & M_ZERO)) {
|
2003-02-01 10:07:49 +00:00
|
|
|
memset(va, 0x70, osize);
|
|
|
|
}
|
|
|
|
#endif
|
1994-05-24 10:09:53 +00:00
|
|
|
return ((void *) va);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
1999-01-21 08:29:12 +00:00
|
|
|
* free:
|
|
|
|
*
|
|
|
|
* Free a block of memory allocated by malloc.
|
|
|
|
*
|
|
|
|
* This routine may not block.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
void
|
2005-04-12 23:54:34 +00:00
|
|
|
free(void *addr, struct malloc_type *type)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-03-19 09:11:49 +00:00
|
|
|
uma_slab_t slab;
|
|
|
|
u_long size;
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/* free(NULL, ...) does nothing */
|
|
|
|
if (addr == NULL)
|
|
|
|
return;
|
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
/* XXX CHANGEME! */
|
|
|
|
if (type == M_SUBPROC) {
|
|
|
|
memguard_free(addr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
Reimplement contigmalloc(9) with an algorithm which stands a greatly-
improved chance of working despite pressure from running programs.
Instead of trying to throw a bunch of pages out to swap and hope for
the best, only a range that can potentially fulfill contigmalloc(9)'s
request will have its contents paged out (potentially, not forcibly)
at a time.
The new contigmalloc operation still operates in three passes, but it
could potentially be tuned to more or less. The first pass only looks
at pages in the cache and free pages, so they would be thrown out
without having to block. If this is not enough, the subsequent passes
page out any unwired memory. To combat memory pressure refragmenting
the section of memory being laundered, each page is removed from the
systems' free memory queue once it has been freed so that blocking
later doesn't cause the memory laundered so far to get reallocated.
The page-out operations are now blocking, as it would make little sense
to try to push out a page, then get its status immediately afterward
to remove it from the available free pages queue, if it's unlikely to
have been freed. Another change is that if KVA allocation fails, the
allocated memory segment will be freed and not leaked.
There is a sysctl/tunable, defaulting to on, which causes the old
contigmalloc() algorithm to be used. Nonetheless, I have been using
vm.old_contigmalloc=0 for over a month. It is safe to switch at
run-time to see the difference it makes.
A new interface has been used which does not require mapping the
allocated pages into KVA: vm_page.h functions vm_page_alloc_contig()
and vm_page_release_contig(). These are what vm.old_contigmalloc=0
uses internally, so the sysctl/tunable does not affect their operation.
When using the contigmalloc(9) and contigfree(9) interfaces, memory
is now tracked with malloc(9) stats. Several functions have been
exported from kern_malloc.c to allow other subsystems to use these
statistics, as well. This invalidates the BUGS section of the
contigmalloc(9) manpage.
2004-07-19 06:21:27 +00:00
|
|
|
KASSERT(type->ks_memuse > 0,
|
2003-05-05 08:32:53 +00:00
|
|
|
("malloc(9)/free(9) confusion.\n%s",
|
|
|
|
"Probably freeing with wrong type, but maybe not here."));
|
2002-03-19 09:11:49 +00:00
|
|
|
size = 0;
|
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
|
2002-03-19 09:11:49 +00:00
|
|
|
|
|
|
|
if (slab == NULL)
|
2002-04-15 04:05:53 +00:00
|
|
|
panic("free: address %p(%p) has not been allocated.\n",
|
2002-09-18 08:26:30 +00:00
|
|
|
addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
|
|
|
|
if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
|
2002-05-02 09:07:04 +00:00
|
|
|
#ifdef INVARIANTS
|
|
|
|
struct malloc_type **mtp = addr;
|
|
|
|
#endif
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
size = slab->us_keg->uk_size;
|
2002-05-02 09:07:04 +00:00
|
|
|
#ifdef INVARIANTS
|
|
|
|
/*
|
|
|
|
* Cache a pointer to the malloc_type that most recently freed
|
|
|
|
* this memory here. This way we know who is most likely to
|
|
|
|
* have stepped on it later.
|
|
|
|
*
|
|
|
|
* This code assumes that size is a multiple of 8 bytes for
|
|
|
|
* 64 bit machines
|
|
|
|
*/
|
|
|
|
mtp = (struct malloc_type **)
|
|
|
|
((unsigned long)mtp & ~UMA_ALIGN_PTR);
|
|
|
|
mtp += (size - sizeof(struct malloc_type *)) /
|
|
|
|
sizeof(struct malloc_type *);
|
|
|
|
*mtp = type;
|
|
|
|
#endif
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab);
|
2002-03-19 09:11:49 +00:00
|
|
|
} else {
|
|
|
|
size = slab->us_size;
|
|
|
|
uma_large_free(slab);
|
1999-05-06 18:13:11 +00:00
|
|
|
}
|
Reimplement contigmalloc(9) with an algorithm which stands a greatly-
improved chance of working despite pressure from running programs.
Instead of trying to throw a bunch of pages out to swap and hope for
the best, only a range that can potentially fulfill contigmalloc(9)'s
request will have its contents paged out (potentially, not forcibly)
at a time.
The new contigmalloc operation still operates in three passes, but it
could potentially be tuned to more or less. The first pass only looks
at pages in the cache and free pages, so they would be thrown out
without having to block. If this is not enough, the subsequent passes
page out any unwired memory. To combat memory pressure refragmenting
the section of memory being laundered, each page is removed from the
systems' free memory queue once it has been freed so that blocking
later doesn't cause the memory laundered so far to get reallocated.
The page-out operations are now blocking, as it would make little sense
to try to push out a page, then get its status immediately afterward
to remove it from the available free pages queue, if it's unlikely to
have been freed. Another change is that if KVA allocation fails, the
allocated memory segment will be freed and not leaked.
There is a sysctl/tunable, defaulting to on, which causes the old
contigmalloc() algorithm to be used. Nonetheless, I have been using
vm.old_contigmalloc=0 for over a month. It is safe to switch at
run-time to see the difference it makes.
A new interface has been used which does not require mapping the
allocated pages into KVA: vm_page.h functions vm_page_alloc_contig()
and vm_page_release_contig(). These are what vm.old_contigmalloc=0
uses internally, so the sysctl/tunable does not affect their operation.
When using the contigmalloc(9) and contigfree(9) interfaces, memory
is now tracked with malloc(9) stats. Several functions have been
exported from kern_malloc.c to allow other subsystems to use these
statistics, as well. This invalidates the BUGS section of the
contigmalloc(9) manpage.
2004-07-19 06:21:27 +00:00
|
|
|
malloc_type_freed(type, size);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/*
|
|
|
|
* realloc: change the size of a memory block
|
|
|
|
*/
|
|
|
|
void *
|
2005-04-12 23:54:34 +00:00
|
|
|
realloc(void *addr, unsigned long size, struct malloc_type *type, int flags)
|
2002-03-13 01:42:33 +00:00
|
|
|
{
|
2002-03-19 09:11:49 +00:00
|
|
|
uma_slab_t slab;
|
2002-03-13 01:42:33 +00:00
|
|
|
unsigned long alloc;
|
|
|
|
void *newaddr;
|
|
|
|
|
|
|
|
/* realloc(NULL, ...) is equivalent to malloc(...) */
|
|
|
|
if (addr == NULL)
|
|
|
|
return (malloc(size, type, flags));
|
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
/* XXX: CHANGEME! */
|
|
|
|
if (type == M_SUBPROC) {
|
|
|
|
slab = NULL;
|
|
|
|
alloc = size;
|
|
|
|
} else {
|
|
|
|
#endif
|
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
slab = vtoslab((vm_offset_t)addr & ~(UMA_SLAB_MASK));
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/* Sanity check */
|
2002-03-19 09:11:49 +00:00
|
|
|
KASSERT(slab != NULL,
|
2002-03-13 01:42:33 +00:00
|
|
|
("realloc: address %p out of range", (void *)addr));
|
|
|
|
|
|
|
|
/* Get the size of the original block */
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
if (slab->us_keg)
|
|
|
|
alloc = slab->us_keg->uk_size;
|
2002-03-19 09:11:49 +00:00
|
|
|
else
|
|
|
|
alloc = slab->us_size;
|
2002-03-13 01:42:33 +00:00
|
|
|
|
|
|
|
/* Reuse the original block if appropriate */
|
|
|
|
if (size <= alloc
|
|
|
|
&& (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE))
|
|
|
|
return (addr);
|
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/* Allocate a new, bigger (or smaller) block */
|
|
|
|
if ((newaddr = malloc(size, type, flags)) == NULL)
|
|
|
|
return (NULL);
|
|
|
|
|
|
|
|
/* Copy over original contents */
|
|
|
|
bcopy(addr, newaddr, min(size, alloc));
|
|
|
|
free(addr, type);
|
|
|
|
return (newaddr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* reallocf: same as realloc() but free memory on failure.
|
|
|
|
*/
|
|
|
|
void *
|
2005-04-12 23:54:34 +00:00
|
|
|
reallocf(void *addr, unsigned long size, struct malloc_type *type, int flags)
|
2002-03-13 01:42:33 +00:00
|
|
|
{
|
|
|
|
void *mem;
|
|
|
|
|
2003-07-22 10:36:36 +00:00
|
|
|
if ((mem = realloc(addr, size, type, flags)) == NULL)
|
2002-03-13 01:42:33 +00:00
|
|
|
free(addr, type);
|
|
|
|
return (mem);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Initialize the kernel memory allocator
|
|
|
|
*/
|
1995-08-28 09:19:25 +00:00
|
|
|
/* ARGSUSED*/
|
|
|
|
static void
|
2005-04-12 23:54:34 +00:00
|
|
|
kmeminit(void *dummy)
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-04-15 04:05:53 +00:00
|
|
|
u_int8_t indx;
|
2000-01-28 04:04:58 +00:00
|
|
|
u_long mem_size;
|
2002-03-19 09:11:49 +00:00
|
|
|
int i;
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-04-04 21:03:38 +00:00
|
|
|
mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF);
|
2000-09-11 02:32:30 +00:00
|
|
|
|
1998-02-23 07:41:23 +00:00
|
|
|
/*
|
|
|
|
* Try to auto-tune the kernel memory size, so that it is
|
|
|
|
* more applicable for a wider range of machine sizes.
|
|
|
|
* On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while
|
|
|
|
* a VM_KMEM_SIZE of 12MB is a fair compromise. The
|
|
|
|
* VM_KMEM_SIZE_MAX is dependent on the maximum KVA space
|
|
|
|
* available, and on an X86 with a total KVA space of 256MB,
|
|
|
|
* try to keep VM_KMEM_SIZE_MAX at 80MB or below.
|
|
|
|
*
|
|
|
|
* Note that the kmem_map is also used by the zone allocator,
|
|
|
|
* so make sure that there is enough space.
|
|
|
|
*/
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
vm_kmem_size = VM_KMEM_SIZE + nmbclusters * PAGE_SIZE;
|
2003-06-11 05:18:59 +00:00
|
|
|
mem_size = cnt.v_page_count;
|
1998-02-23 07:41:23 +00:00
|
|
|
|
|
|
|
#if defined(VM_KMEM_SIZE_SCALE)
|
2004-09-29 14:21:40 +00:00
|
|
|
vm_kmem_size_scale = VM_KMEM_SIZE_SCALE;
|
1998-02-23 07:41:23 +00:00
|
|
|
#endif
|
2004-09-29 14:21:40 +00:00
|
|
|
TUNABLE_INT_FETCH("vm.kmem_size_scale", &vm_kmem_size_scale);
|
|
|
|
if (vm_kmem_size_scale > 0 &&
|
|
|
|
(mem_size / vm_kmem_size_scale) > (vm_kmem_size / PAGE_SIZE))
|
|
|
|
vm_kmem_size = (mem_size / vm_kmem_size_scale) * PAGE_SIZE;
|
1998-02-23 07:41:23 +00:00
|
|
|
|
|
|
|
#if defined(VM_KMEM_SIZE_MAX)
|
2004-09-29 14:21:40 +00:00
|
|
|
vm_kmem_size_max = VM_KMEM_SIZE_MAX;
|
1998-02-23 07:41:23 +00:00
|
|
|
#endif
|
2004-09-29 14:21:40 +00:00
|
|
|
TUNABLE_INT_FETCH("vm.kmem_size_max", &vm_kmem_size_max);
|
|
|
|
if (vm_kmem_size_max > 0 && vm_kmem_size >= vm_kmem_size_max)
|
|
|
|
vm_kmem_size = vm_kmem_size_max;
|
1998-02-23 07:41:23 +00:00
|
|
|
|
1999-01-21 21:54:32 +00:00
|
|
|
/* Allow final override from the kernel environment */
|
2004-01-27 15:59:38 +00:00
|
|
|
#ifndef BURN_BRIDGES
|
|
|
|
if (TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size) != 0)
|
|
|
|
printf("kern.vm.kmem.size is now called vm.kmem_size!\n");
|
|
|
|
#endif
|
|
|
|
TUNABLE_INT_FETCH("vm.kmem_size", &vm_kmem_size);
|
1999-01-21 21:54:32 +00:00
|
|
|
|
2000-01-28 04:04:58 +00:00
|
|
|
/*
|
|
|
|
* Limit kmem virtual size to twice the physical memory.
|
|
|
|
* This allows for kmem map sparseness, but limits the size
|
|
|
|
* to something sane. Be careful to not overflow the 32bit
|
|
|
|
* ints while doing the check.
|
|
|
|
*/
|
2003-06-11 05:18:59 +00:00
|
|
|
if (((vm_kmem_size / 2) / PAGE_SIZE) > cnt.v_page_count)
|
2000-01-28 04:04:58 +00:00
|
|
|
vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE;
|
1998-02-23 07:41:23 +00:00
|
|
|
|
2003-07-11 00:01:03 +00:00
|
|
|
/*
|
|
|
|
* Tune settings based on the kernel map's size at this time.
|
|
|
|
*/
|
|
|
|
init_param3(vm_kmem_size / PAGE_SIZE);
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase,
|
Bring in mbuma to replace mballoc.
mbuma is an Mbuf & Cluster allocator built on top of a number of
extensions to the UMA framework, all included herein.
Extensions to UMA worth noting:
- Better layering between slab <-> zone caches; introduce
Keg structure which splits off slab cache away from the
zone structure and allows multiple zones to be stacked
on top of a single Keg (single type of slab cache);
perhaps we should look into defining a subset API on
top of the Keg for special use by malloc(9),
for example.
- UMA_ZONE_REFCNT zones can now be added, and reference
counters automagically allocated for them within the end
of the associated slab structures. uma_find_refcnt()
does a kextract to fetch the slab struct reference from
the underlying page, and lookup the corresponding refcnt.
mbuma things worth noting:
- integrates mbuf & cluster allocations with extended UMA
and provides caches for commonly-allocated items; defines
several zones (two primary, one secondary) and two kegs.
- change up certain code paths that always used to do:
m_get() + m_clget() to instead just use m_getcl() and
try to take advantage of the newly defined secondary
Packet zone.
- netstat(1) and systat(1) quickly hacked up to do basic
stat reporting but additional stats work needs to be
done once some other details within UMA have been taken
care of and it becomes clearer to how stats will work
within the modified framework.
From the user perspective, one implication is that the
NMBCLUSTERS compile-time option is no longer used. The
maximum number of clusters is still capped off according
to maxusers, but it can be made unlimited by setting
the kern.ipc.nmbclusters boot-time tunable to zero.
Work should be done to write an appropriate sysctl
handler allowing dynamic tuning of kern.ipc.nmbclusters
at runtime.
Additional things worth noting/known issues (READ):
- One report of 'ips' (ServeRAID) driver acting really
slow in conjunction with mbuma. Need more data.
Latest report is that ips is equally sucking with
and without mbuma.
- Giant leak in NFS code sometimes occurs, can't
reproduce but currently analyzing; brueffer is
able to reproduce but THIS IS NOT an mbuma-specific
problem and currently occurs even WITHOUT mbuma.
- Issues in network locking: there is at least one
code path in the rip code where one or more locks
are acquired and we end up in m_prepend() with
M_WAITOK, which causes WITNESS to whine from within
UMA. Current temporary solution: force all UMA
allocations to be M_NOWAIT from within UMA for now
to avoid deadlocks unless WITNESS is defined and we
can determine with certainty that we're not holding
any locks when we're M_WAITOK.
- I've seen at least one weird socketbuffer empty-but-
mbuf-still-attached panic. I don't believe this
to be related to mbuma but please keep your eyes
open, turn on debugging, and capture crash dumps.
This change removes more code than it adds.
A paper is available detailing the change and considering
various performance issues, it was presented at BSDCan2004:
http://www.unixdaemons.com/~bmilekic/netbuf_bmilekic.pdf
Please read the paper for Future Work and implementation
details, as well as credits.
Testing and Debugging:
rwatson,
brueffer,
Ketrien I. Saihr-Kesenchedra,
...
Reviewed by: Lots of people (for different parts)
2004-05-31 21:46:06 +00:00
|
|
|
(vm_offset_t *)&kmemlimit, vm_kmem_size);
|
1997-08-05 00:02:08 +00:00
|
|
|
kmem_map->system_map = 1;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2005-01-21 18:09:17 +00:00
|
|
|
#ifdef DEBUG_MEMGUARD
|
|
|
|
/*
|
|
|
|
* Initialize MemGuard if support compiled in. MemGuard is a
|
|
|
|
* replacement allocator used for detecting tamper-after-free
|
|
|
|
* scenarios as they occur. It is only used for debugging.
|
|
|
|
*/
|
|
|
|
vm_memguard_divisor = 10;
|
|
|
|
TUNABLE_INT_FETCH("vm.memguard_divisor", &vm_memguard_divisor);
|
|
|
|
|
|
|
|
/* Pick a conservative value if provided value sucks. */
|
|
|
|
if ((vm_memguard_divisor <= 0) ||
|
|
|
|
((vm_kmem_size / vm_memguard_divisor) == 0))
|
|
|
|
vm_memguard_divisor = 10;
|
|
|
|
memguard_init(kmem_map, vm_kmem_size / vm_memguard_divisor);
|
|
|
|
#endif
|
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
uma_startup2();
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
|
|
|
|
int size = kmemzones[indx].kz_size;
|
|
|
|
char *name = kmemzones[indx].kz_name;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-04-30 07:54:25 +00:00
|
|
|
kmemzones[indx].kz_zone = uma_zcreate(name, size,
|
|
|
|
#ifdef INVARIANTS
|
2002-05-02 09:07:04 +00:00
|
|
|
mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
|
2002-04-30 07:54:25 +00:00
|
|
|
#else
|
|
|
|
NULL, NULL, NULL, NULL,
|
|
|
|
#endif
|
|
|
|
UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
for (;i <= size; i+= KMEM_ZBASE)
|
2002-04-15 04:05:53 +00:00
|
|
|
kmemsize[i >> KMEM_ZSHIFT] = indx;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1997-10-10 14:06:34 +00:00
|
|
|
}
|
|
|
|
|
1998-11-10 08:46:24 +00:00
|
|
|
void
|
2005-04-12 23:54:34 +00:00
|
|
|
malloc_init(void *data)
|
1997-10-10 14:06:34 +00:00
|
|
|
{
|
1998-11-10 08:46:24 +00:00
|
|
|
struct malloc_type *type = (struct malloc_type *)data;
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
1999-11-22 02:45:11 +00:00
|
|
|
if (type->ks_magic != M_MAGIC)
|
1997-10-28 19:01:02 +00:00
|
|
|
panic("malloc type lacks magic");
|
|
|
|
|
1997-12-05 05:36:58 +00:00
|
|
|
if (cnt.v_page_count == 0)
|
|
|
|
panic("malloc_init not allowed before vm init");
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
if (type->ks_next != NULL)
|
|
|
|
return;
|
|
|
|
|
1997-10-10 14:06:34 +00:00
|
|
|
type->ks_next = kmemstatistics;
|
|
|
|
kmemstatistics = type;
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_init(&type->ks_mtx, type->ks_shortdesc, "Malloc Stats", MTX_DEF);
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1998-11-10 08:46:24 +00:00
|
|
|
|
|
|
|
void
|
2005-04-12 23:54:34 +00:00
|
|
|
malloc_uninit(void *data)
|
1998-11-10 08:46:24 +00:00
|
|
|
{
|
|
|
|
struct malloc_type *type = (struct malloc_type *)data;
|
|
|
|
struct malloc_type *t;
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&type->ks_mtx);
|
1999-11-22 02:45:11 +00:00
|
|
|
if (type->ks_magic != M_MAGIC)
|
1998-11-10 08:46:24 +00:00
|
|
|
panic("malloc type lacks magic");
|
|
|
|
|
|
|
|
if (cnt.v_page_count == 0)
|
|
|
|
panic("malloc_uninit not allowed before vm init");
|
|
|
|
|
|
|
|
if (type == kmemstatistics)
|
|
|
|
kmemstatistics = type->ks_next;
|
|
|
|
else {
|
|
|
|
for (t = kmemstatistics; t->ks_next != NULL; t = t->ks_next) {
|
|
|
|
if (t->ks_next == type) {
|
|
|
|
t->ks_next = type->ks_next;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
1999-05-12 11:11:27 +00:00
|
|
|
type->ks_next = NULL;
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_destroy(&type->ks_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
1998-11-10 08:46:24 +00:00
|
|
|
}
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
static int
|
|
|
|
sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
|
|
|
|
{
|
|
|
|
struct malloc_type *type;
|
|
|
|
int linesize = 128;
|
|
|
|
int curline;
|
|
|
|
int bufsize;
|
|
|
|
int first;
|
|
|
|
int error;
|
|
|
|
char *buf;
|
|
|
|
char *p;
|
|
|
|
int cnt;
|
|
|
|
int len;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
cnt = 0;
|
|
|
|
|
|
|
|
mtx_lock(&malloc_mtx);
|
|
|
|
for (type = kmemstatistics; type != NULL; type = type->ks_next)
|
|
|
|
cnt++;
|
|
|
|
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
bufsize = linesize * (cnt + 1);
|
2003-02-19 05:47:46 +00:00
|
|
|
p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
len = snprintf(p, linesize,
|
|
|
|
"\n Type InUse MemUse HighUse Requests Size(s)\n");
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
for (type = kmemstatistics; cnt != 0 && type != NULL;
|
|
|
|
type = type->ks_next, cnt--) {
|
|
|
|
if (type->ks_calls == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
curline = linesize - 2; /* Leave room for the \n */
|
2002-04-30 22:39:32 +00:00
|
|
|
len = snprintf(p, curline, "%13s%6lu%6luK%7luK%9llu",
|
2002-04-15 04:05:53 +00:00
|
|
|
type->ks_shortdesc,
|
|
|
|
type->ks_inuse,
|
|
|
|
(type->ks_memuse + 1023) / 1024,
|
|
|
|
(type->ks_maxused + 1023) / 1024,
|
|
|
|
(long long unsigned)type->ks_calls);
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
first = 1;
|
2002-05-31 09:41:09 +00:00
|
|
|
for (i = 0; i < sizeof(kmemzones) / sizeof(kmemzones[0]) - 1;
|
|
|
|
i++) {
|
2002-04-15 04:05:53 +00:00
|
|
|
if (type->ks_size & (1 << i)) {
|
|
|
|
if (first)
|
|
|
|
len = snprintf(p, curline, " ");
|
|
|
|
else
|
|
|
|
len = snprintf(p, curline, ",");
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
len = snprintf(p, curline,
|
|
|
|
"%s", kmemzones[i].kz_name);
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
first = 0;
|
|
|
|
}
|
2002-05-31 09:41:09 +00:00
|
|
|
}
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
len = snprintf(p, 2, "\n");
|
|
|
|
p += len;
|
|
|
|
}
|
|
|
|
|
|
|
|
mtx_unlock(&malloc_mtx);
|
|
|
|
error = SYSCTL_OUT(req, buf, p - buf);
|
|
|
|
|
|
|
|
free(buf, M_TEMP);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD,
|
|
|
|
NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats");
|
2002-04-15 05:24:01 +00:00
|
|
|
|
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
|
|
|
|
static int
|
|
|
|
sysctl_kern_mprof(SYSCTL_HANDLER_ARGS)
|
|
|
|
{
|
|
|
|
int linesize = 64;
|
|
|
|
uint64_t count;
|
|
|
|
uint64_t waste;
|
|
|
|
uint64_t mem;
|
|
|
|
int bufsize;
|
|
|
|
int error;
|
|
|
|
char *buf;
|
|
|
|
int rsize;
|
|
|
|
int size;
|
|
|
|
char *p;
|
|
|
|
int len;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
bufsize = linesize * (KMEM_ZSIZE + 1);
|
|
|
|
bufsize += 128; /* For the stats line */
|
|
|
|
bufsize += 128; /* For the banner line */
|
|
|
|
waste = 0;
|
|
|
|
mem = 0;
|
|
|
|
|
2003-02-19 05:47:46 +00:00
|
|
|
p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
|
2002-04-15 05:24:01 +00:00
|
|
|
len = snprintf(p, bufsize,
|
|
|
|
"\n Size Requests Real Size\n");
|
|
|
|
bufsize -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
for (i = 0; i < KMEM_ZSIZE; i++) {
|
|
|
|
size = i << KMEM_ZSHIFT;
|
|
|
|
rsize = kmemzones[kmemsize[i]].kz_size;
|
|
|
|
count = (long long unsigned)krequests[i];
|
|
|
|
|
|
|
|
len = snprintf(p, bufsize, "%6d%28llu%11d\n",
|
|
|
|
size, (unsigned long long)count, rsize);
|
|
|
|
bufsize -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
if ((rsize * count) > (size * count))
|
|
|
|
waste += (rsize * count) - (size * count);
|
|
|
|
mem += (rsize * count);
|
|
|
|
}
|
|
|
|
|
|
|
|
len = snprintf(p, bufsize,
|
|
|
|
"\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n",
|
|
|
|
(unsigned long long)mem, (unsigned long long)waste);
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
error = SYSCTL_OUT(req, buf, p - buf);
|
|
|
|
|
|
|
|
free(buf, M_TEMP);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD,
|
|
|
|
NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling");
|
|
|
|
#endif /* MALLOC_PROFILE */
|