mirror of
https://git.FreeBSD.org/src.git
synced 2024-12-29 12:03:03 +00:00
1545 lines
47 KiB
C
1545 lines
47 KiB
C
/*-
|
|
* Copyright (c) 2001, 2002, 2003
|
|
* Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. The name of the author may not be used to endorse or promote products
|
|
* derived from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include "opt_mac.h"
|
|
#include "opt_param.h"
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/mac.h>
|
|
#include <sys/mbuf.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/condvar.h>
|
|
#include <sys/smp.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/sysctl.h>
|
|
#include <sys/domain.h>
|
|
#include <sys/protosw.h>
|
|
|
|
#include <vm/vm.h>
|
|
#include <vm/vm_kern.h>
|
|
#include <vm/vm_extern.h>
|
|
#include <vm/pmap.h>
|
|
#include <vm/vm_map.h>
|
|
|
|
/*
|
|
* mb_alloc: network buffer allocator
|
|
*
|
|
* XXX: currently, the "low watermark" sysctl is marked read-only as its
|
|
* effects are not completely implemented. To be fixed soon.
|
|
*/
|
|
|
|
/*
|
|
* Maximum number of PCPU containers. If you know what you're doing you could
|
|
* explicitly define MBALLOC_NCPU to be exactly the number of CPUs on your
|
|
* system during compilation, and thus prevent kernel structure bloat.
|
|
*
|
|
* SMP and non-SMP kernels clearly have a different number of possible CPUs,
|
|
* but because we cannot assume a dense array of CPUs, we always allocate
|
|
* and traverse PCPU containers up to NCPU amount and merely check for
|
|
* CPU availability.
|
|
*/
|
|
#ifdef MBALLOC_NCPU
|
|
#define NCPU MBALLOC_NCPU
|
|
#else
|
|
#define NCPU MAXCPU
|
|
#endif
|
|
|
|
/*-
|
|
* The mbuf allocator is based on Alfred Perlstein's <alfred@FreeBSD.org>
|
|
* "memcache" proof-of-concept allocator which was itself based on
|
|
* several well-known SMP-friendly allocators.
|
|
*
|
|
* The mb_alloc mbuf allocator is a special when compared to other
|
|
* general-purpose allocators. Some things to take note of:
|
|
*
|
|
* Mbufs and mbuf clusters are two different objects. Sometimes we
|
|
* will allocate a single mbuf, other times a single cluster,
|
|
* other times both. Further, we may sometimes wish to allocate a
|
|
* whole chain of mbufs with clusters. This allocator will perform
|
|
* the common case of each scenario in one function call (this
|
|
* includes constructing or destructing the object) while only
|
|
* locking/unlocking the cache once, if it can get away with it.
|
|
* The caches consist of pure mbufs and pure clusters; that is
|
|
* there are no 'zones' containing mbufs with already pre-hooked
|
|
* clusters. Since we can allocate both objects atomically anyway,
|
|
* we don't bother fragmenting our caches for any particular 'scenarios.'
|
|
*
|
|
* We allocate from seperate sub-maps of kmem_map, thus imposing
|
|
* an ultimate upper-limit on the number of allocatable clusters
|
|
* and mbufs and also, since the clusters all come from a
|
|
* virtually contiguous region, we can keep reference counters
|
|
* for them and "allocate" them purely by indexing into a
|
|
* dense refcount vector.
|
|
*
|
|
* We call out to protocol drain routines (which can be hooked
|
|
* into us) when we're low on space.
|
|
*
|
|
* The mbuf allocator keeps all objects that it allocates in mb_buckets.
|
|
* The buckets keep a number of objects (an object can be an mbuf or an
|
|
* mbuf cluster) and facilitate moving larger sets of contiguous objects
|
|
* from the per-CPU caches to the global cache. The buckets also have
|
|
* the added advantage that objects, when migrated from cache to cache,
|
|
* are migrated in chunks that keep contiguous objects together,
|
|
* minimizing TLB pollution.
|
|
*
|
|
* The buckets are kept on singly-linked lists called "containers." A container
|
|
* is protected by a mutex in order to ensure consistency. The mutex
|
|
* itself is allocated separately and attached to the container at boot time,
|
|
* thus allowing for certain containers to share the same lock. Per-CPU
|
|
* containers for mbufs and mbuf clusters all share the same per-CPU
|
|
* lock whereas the global cache containers for these objects share one
|
|
* global lock.
|
|
*/
|
|
struct mb_bucket {
|
|
SLIST_ENTRY(mb_bucket) mb_blist;
|
|
int mb_owner;
|
|
int mb_numfree;
|
|
void *mb_free[0];
|
|
};
|
|
|
|
struct mb_container {
|
|
SLIST_HEAD(mc_buckethd, mb_bucket) mc_bhead;
|
|
struct mtx *mc_lock;
|
|
int mc_numowner;
|
|
u_int mc_starved;
|
|
long *mc_types;
|
|
u_long *mc_objcount;
|
|
u_long *mc_numbucks;
|
|
};
|
|
|
|
struct mb_gen_list {
|
|
struct mb_container mb_cont;
|
|
struct cv mgl_mstarved;
|
|
};
|
|
|
|
struct mb_pcpu_list {
|
|
struct mb_container mb_cont;
|
|
};
|
|
|
|
/*
|
|
* Boot-time configurable object counts that will determine the maximum
|
|
* number of permitted objects in the mbuf and mcluster cases. In the
|
|
* ext counter (nmbcnt) case, it's just an indicator serving to scale
|
|
* kmem_map size properly - in other words, we may be allowed to allocate
|
|
* more than nmbcnt counters, whereas we will never be allowed to allocate
|
|
* more than nmbufs mbufs or nmbclusters mclusters.
|
|
* As for nsfbufs, it is used to indicate how many sendfile(2) buffers will be
|
|
* allocatable by the sfbuf allocator (found in uipc_syscalls.c)
|
|
*/
|
|
#ifndef NMBCLUSTERS
|
|
#define NMBCLUSTERS (1024 + maxusers * 64)
|
|
#endif
|
|
#ifndef NMBUFS
|
|
#define NMBUFS (nmbclusters * 2)
|
|
#endif
|
|
#ifndef NSFBUFS
|
|
#define NSFBUFS (512 + maxusers * 16)
|
|
#endif
|
|
#ifndef NMBCNTS
|
|
#define NMBCNTS (nmbclusters + nsfbufs)
|
|
#endif
|
|
int nmbufs;
|
|
int nmbclusters;
|
|
int nmbcnt;
|
|
int nsfbufs;
|
|
|
|
/*
|
|
* Sizes of objects per bucket. There are this size's worth of mbufs
|
|
* or clusters in each bucket. Please keep these a power-of-2.
|
|
*/
|
|
#define MBUF_BUCK_SZ (PAGE_SIZE * 2)
|
|
#define CLUST_BUCK_SZ (PAGE_SIZE * 4)
|
|
|
|
/*
|
|
* Perform sanity checks of tunables declared above.
|
|
*/
|
|
static void
|
|
tunable_mbinit(void *dummy)
|
|
{
|
|
|
|
/*
|
|
* This has to be done before VM init.
|
|
*/
|
|
nmbclusters = NMBCLUSTERS;
|
|
TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
|
|
nmbufs = NMBUFS;
|
|
TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
|
|
nsfbufs = NSFBUFS;
|
|
TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
|
|
nmbcnt = NMBCNTS;
|
|
TUNABLE_INT_FETCH("kern.ipc.nmbcnt", &nmbcnt);
|
|
/* Sanity checks */
|
|
if (nmbufs < nmbclusters * 2)
|
|
nmbufs = nmbclusters * 2;
|
|
if (nmbcnt < nmbclusters + nsfbufs)
|
|
nmbcnt = nmbclusters + nsfbufs;
|
|
}
|
|
SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
|
|
|
|
/*
|
|
* The freelist structures and mutex locks. The number statically declared
|
|
* here depends on the number of CPUs.
|
|
*
|
|
* We set up in such a way that all the objects (mbufs, clusters)
|
|
* share the same mutex lock. It has been established that we do not benefit
|
|
* from different locks for different objects, so we use the same lock,
|
|
* regardless of object type. This also allows us to do optimised
|
|
* multi-object allocations without dropping the lock in between.
|
|
*/
|
|
struct mb_lstmngr {
|
|
struct mb_gen_list *ml_genlist;
|
|
struct mb_pcpu_list *ml_cntlst[NCPU];
|
|
struct mb_bucket **ml_btable;
|
|
vm_map_t ml_map;
|
|
vm_offset_t ml_mapbase;
|
|
vm_offset_t ml_maptop;
|
|
int ml_mapfull;
|
|
u_int ml_objsize;
|
|
u_int ml_objbucks;
|
|
u_int *ml_wmhigh;
|
|
u_int *ml_wmlow;
|
|
};
|
|
static struct mb_lstmngr mb_list_mbuf, mb_list_clust;
|
|
static struct mtx mbuf_gen, mbuf_pcpu[NCPU];
|
|
static u_int *cl_refcntmap;
|
|
|
|
/*
|
|
* Local macros for internal allocator structure manipulations.
|
|
*/
|
|
#ifdef SMP
|
|
#define MB_GET_PCPU_LIST(mb_lst) (mb_lst)->ml_cntlst[PCPU_GET(cpuid)]
|
|
#else
|
|
#define MB_GET_PCPU_LIST(mb_lst) (mb_lst)->ml_cntlst[0]
|
|
#endif
|
|
|
|
#define MB_GET_GEN_LIST(mb_lst) (mb_lst)->ml_genlist
|
|
|
|
#define MB_LOCK_CONT(mb_cnt) mtx_lock((mb_cnt)->mb_cont.mc_lock)
|
|
|
|
#define MB_UNLOCK_CONT(mb_cnt) mtx_unlock((mb_cnt)->mb_cont.mc_lock)
|
|
|
|
#define MB_GET_PCPU_LIST_NUM(mb_lst, num) \
|
|
(mb_lst)->ml_cntlst[(num)]
|
|
|
|
#define MB_BUCKET_INDX(mb_obj, mb_lst) \
|
|
(int)(((caddr_t)(mb_obj) - (caddr_t)(mb_lst)->ml_mapbase) / \
|
|
((mb_lst)->ml_objbucks * (mb_lst)->ml_objsize))
|
|
|
|
#define MB_GET_OBJECT(mb_objp, mb_bckt, mb_lst) \
|
|
{ \
|
|
struct mc_buckethd *_mchd = &((mb_lst)->mb_cont.mc_bhead); \
|
|
\
|
|
(mb_bckt)->mb_numfree--; \
|
|
(mb_objp) = (mb_bckt)->mb_free[((mb_bckt)->mb_numfree)]; \
|
|
(*((mb_lst)->mb_cont.mc_objcount))--; \
|
|
if ((mb_bckt)->mb_numfree == 0) { \
|
|
SLIST_REMOVE_HEAD(_mchd, mb_blist); \
|
|
SLIST_NEXT((mb_bckt), mb_blist) = NULL; \
|
|
(mb_bckt)->mb_owner |= MB_BUCKET_FREE; \
|
|
} \
|
|
}
|
|
|
|
#define MB_PUT_OBJECT(mb_objp, mb_bckt, mb_lst) \
|
|
(mb_bckt)->mb_free[((mb_bckt)->mb_numfree)] = (mb_objp); \
|
|
(mb_bckt)->mb_numfree++; \
|
|
(*((mb_lst)->mb_cont.mc_objcount))++;
|
|
|
|
#define MB_MBTYPES_INC(mb_cnt, mb_type, mb_num) \
|
|
if ((mb_type) != MT_NOTMBUF) \
|
|
(*((mb_cnt)->mb_cont.mc_types + (mb_type))) += (mb_num)
|
|
|
|
#define MB_MBTYPES_DEC(mb_cnt, mb_type, mb_num) \
|
|
if ((mb_type) != MT_NOTMBUF) \
|
|
(*((mb_cnt)->mb_cont.mc_types + (mb_type))) -= (mb_num)
|
|
|
|
/*
|
|
* Ownership of buckets/containers is represented by integers. The PCPU
|
|
* lists range from 0 to NCPU-1. We need a free numerical id for the general
|
|
* list (we use NCPU). We also need a non-conflicting free bit to indicate
|
|
* that the bucket is free and removed from a container, while not losing
|
|
* the bucket's originating container id. We use the highest bit
|
|
* for the free marker.
|
|
*/
|
|
#define MB_GENLIST_OWNER (NCPU)
|
|
#define MB_BUCKET_FREE (1 << (sizeof(int) * 8 - 1))
|
|
|
|
/* Statistics structures for allocator (per-CPU and general). */
|
|
static struct mbpstat mb_statpcpu[NCPU + 1];
|
|
struct mbstat mbstat;
|
|
|
|
/* Sleep time for wait code (in ticks). */
|
|
static int mbuf_wait = 64;
|
|
|
|
static u_int mbuf_hiwm = 512; /* High wm on # of mbufs per cache */
|
|
static u_int mbuf_lowm = 128; /* Low wm on # of mbufs per cache */
|
|
static u_int clust_hiwm = 128; /* High wm on # of clusters per cache */
|
|
static u_int clust_lowm = 16; /* Low wm on # of clusters per cache */
|
|
|
|
/*
|
|
* Objects exported by sysctl(8).
|
|
*/
|
|
SYSCTL_DECL(_kern_ipc);
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RD, &nmbclusters, 0,
|
|
"Maximum number of mbuf clusters available");
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, nmbufs, CTLFLAG_RD, &nmbufs, 0,
|
|
"Maximum number of mbufs available");
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, nmbcnt, CTLFLAG_RD, &nmbcnt, 0,
|
|
"Number used to scale kmem_map to ensure sufficient space for counters");
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0,
|
|
"Maximum number of sendfile(2) sf_bufs available");
|
|
SYSCTL_INT(_kern_ipc, OID_AUTO, mbuf_wait, CTLFLAG_RW, &mbuf_wait, 0,
|
|
"Sleep time of mbuf subsystem wait allocations during exhaustion");
|
|
SYSCTL_UINT(_kern_ipc, OID_AUTO, mbuf_hiwm, CTLFLAG_RW, &mbuf_hiwm, 0,
|
|
"Upper limit of number of mbufs allowed in each cache");
|
|
SYSCTL_UINT(_kern_ipc, OID_AUTO, mbuf_lowm, CTLFLAG_RD, &mbuf_lowm, 0,
|
|
"Lower limit of number of mbufs allowed in each cache");
|
|
SYSCTL_UINT(_kern_ipc, OID_AUTO, clust_hiwm, CTLFLAG_RW, &clust_hiwm, 0,
|
|
"Upper limit of number of mbuf clusters allowed in each cache");
|
|
SYSCTL_UINT(_kern_ipc, OID_AUTO, clust_lowm, CTLFLAG_RD, &clust_lowm, 0,
|
|
"Lower limit of number of mbuf clusters allowed in each cache");
|
|
SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
|
|
"Mbuf general information and statistics");
|
|
SYSCTL_OPAQUE(_kern_ipc, OID_AUTO, mb_statpcpu, CTLFLAG_RD, mb_statpcpu,
|
|
sizeof(mb_statpcpu), "S,", "Mbuf allocator per CPU statistics");
|
|
|
|
/*
|
|
* Prototypes of local allocator routines.
|
|
*/
|
|
static void *mb_alloc_wait(struct mb_lstmngr *, short);
|
|
static struct mb_bucket *mb_pop_cont(struct mb_lstmngr *, int,
|
|
struct mb_pcpu_list *);
|
|
static void mb_reclaim(void);
|
|
static void mbuf_init(void *);
|
|
|
|
/*
|
|
* Initial allocation numbers. Each parameter represents the number of buckets
|
|
* of each object that will be placed initially in each PCPU container for
|
|
* said object.
|
|
*/
|
|
#define NMB_MBUF_INIT 2
|
|
#define NMB_CLUST_INIT 8
|
|
|
|
/*
|
|
* Internal flags that allow for cache locks to remain "persistent" across
|
|
* allocation and free calls. They may be used in combination.
|
|
*/
|
|
#define MBP_PERSIST 0x1 /* Return with lock still held. */
|
|
#define MBP_PERSISTENT 0x2 /* Cache lock is already held coming in. */
|
|
|
|
/*
|
|
* Initialize the mbuf subsystem.
|
|
*
|
|
* We sub-divide the kmem_map into several submaps; this way, we don't have
|
|
* to worry about artificially limiting the number of mbuf or mbuf cluster
|
|
* allocations, due to fear of one type of allocation "stealing" address
|
|
* space initially reserved for another.
|
|
*
|
|
* Set up both the general containers and all the PCPU containers. Populate
|
|
* the PCPU containers with initial numbers.
|
|
*/
|
|
MALLOC_DEFINE(M_MBUF, "mbufmgr", "mbuf subsystem management structures");
|
|
SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
|
|
static void
|
|
mbuf_init(void *dummy)
|
|
{
|
|
struct mb_pcpu_list *pcpu_cnt;
|
|
vm_size_t mb_map_size;
|
|
int i, j;
|
|
|
|
/*
|
|
* Set up all the submaps, for each type of object that we deal
|
|
* with in this allocator.
|
|
*/
|
|
mb_map_size = (vm_size_t)(nmbufs * MSIZE);
|
|
mb_map_size = rounddown(mb_map_size, MBUF_BUCK_SZ);
|
|
mb_list_mbuf.ml_btable = malloc((unsigned long)mb_map_size /
|
|
MBUF_BUCK_SZ * sizeof(struct mb_bucket *), M_MBUF, M_NOWAIT);
|
|
if (mb_list_mbuf.ml_btable == NULL)
|
|
goto bad;
|
|
mb_list_mbuf.ml_map = kmem_suballoc(kmem_map,&(mb_list_mbuf.ml_mapbase),
|
|
&(mb_list_mbuf.ml_maptop), mb_map_size);
|
|
mb_list_mbuf.ml_map->system_map = 1;
|
|
mb_list_mbuf.ml_mapfull = 0;
|
|
mb_list_mbuf.ml_objsize = MSIZE;
|
|
mb_list_mbuf.ml_objbucks = MBUF_BUCK_SZ / MSIZE;
|
|
mb_list_mbuf.ml_wmhigh = &mbuf_hiwm;
|
|
mb_list_mbuf.ml_wmlow = &mbuf_lowm;
|
|
|
|
mb_map_size = (vm_size_t)(nmbclusters * MCLBYTES);
|
|
mb_map_size = rounddown(mb_map_size, CLUST_BUCK_SZ);
|
|
mb_list_clust.ml_btable = malloc((unsigned long)mb_map_size /
|
|
CLUST_BUCK_SZ * sizeof(struct mb_bucket *), M_MBUF, M_NOWAIT);
|
|
if (mb_list_clust.ml_btable == NULL)
|
|
goto bad;
|
|
mb_list_clust.ml_map = kmem_suballoc(kmem_map,
|
|
&(mb_list_clust.ml_mapbase), &(mb_list_clust.ml_maptop),
|
|
mb_map_size);
|
|
mb_list_clust.ml_map->system_map = 1;
|
|
mb_list_clust.ml_mapfull = 0;
|
|
mb_list_clust.ml_objsize = MCLBYTES;
|
|
mb_list_clust.ml_objbucks = CLUST_BUCK_SZ / MCLBYTES;
|
|
mb_list_clust.ml_wmhigh = &clust_hiwm;
|
|
mb_list_clust.ml_wmlow = &clust_lowm;
|
|
|
|
/*
|
|
* Allocate required general (global) containers for each object type.
|
|
*/
|
|
mb_list_mbuf.ml_genlist = malloc(sizeof(struct mb_gen_list), M_MBUF,
|
|
M_NOWAIT);
|
|
mb_list_clust.ml_genlist = malloc(sizeof(struct mb_gen_list), M_MBUF,
|
|
M_NOWAIT);
|
|
if ((mb_list_mbuf.ml_genlist == NULL) ||
|
|
(mb_list_clust.ml_genlist == NULL))
|
|
goto bad;
|
|
|
|
/*
|
|
* Initialize condition variables and general container mutex locks.
|
|
*/
|
|
mtx_init(&mbuf_gen, "mbuf subsystem general lists lock", NULL, 0);
|
|
cv_init(&(mb_list_mbuf.ml_genlist->mgl_mstarved), "mbuf pool starved");
|
|
cv_init(&(mb_list_clust.ml_genlist->mgl_mstarved),
|
|
"mcluster pool starved");
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_lock =
|
|
mb_list_clust.ml_genlist->mb_cont.mc_lock = &mbuf_gen;
|
|
|
|
/*
|
|
* Set up the general containers for each object.
|
|
*/
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_numowner =
|
|
mb_list_clust.ml_genlist->mb_cont.mc_numowner = MB_GENLIST_OWNER;
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_starved =
|
|
mb_list_clust.ml_genlist->mb_cont.mc_starved = 0;
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_objcount =
|
|
&(mb_statpcpu[MB_GENLIST_OWNER].mb_mbfree);
|
|
mb_list_clust.ml_genlist->mb_cont.mc_objcount =
|
|
&(mb_statpcpu[MB_GENLIST_OWNER].mb_clfree);
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_numbucks =
|
|
&(mb_statpcpu[MB_GENLIST_OWNER].mb_mbbucks);
|
|
mb_list_clust.ml_genlist->mb_cont.mc_numbucks =
|
|
&(mb_statpcpu[MB_GENLIST_OWNER].mb_clbucks);
|
|
mb_list_mbuf.ml_genlist->mb_cont.mc_types =
|
|
&(mb_statpcpu[MB_GENLIST_OWNER].mb_mbtypes[0]);
|
|
mb_list_clust.ml_genlist->mb_cont.mc_types = NULL;
|
|
SLIST_INIT(&(mb_list_mbuf.ml_genlist->mb_cont.mc_bhead));
|
|
SLIST_INIT(&(mb_list_clust.ml_genlist->mb_cont.mc_bhead));
|
|
|
|
/*
|
|
* Allocate all the required counters for clusters. This makes
|
|
* cluster allocations/deallocations much faster.
|
|
*/
|
|
cl_refcntmap = malloc(nmbclusters * sizeof(u_int), M_MBUF, M_NOWAIT);
|
|
if (cl_refcntmap == NULL)
|
|
goto bad;
|
|
|
|
/*
|
|
* Initialize general mbuf statistics.
|
|
*/
|
|
mbstat.m_msize = MSIZE;
|
|
mbstat.m_mclbytes = MCLBYTES;
|
|
mbstat.m_minclsize = MINCLSIZE;
|
|
mbstat.m_mlen = MLEN;
|
|
mbstat.m_mhlen = MHLEN;
|
|
mbstat.m_numtypes = MT_NTYPES;
|
|
mbstat.m_mbperbuck = MBUF_BUCK_SZ / MSIZE;
|
|
mbstat.m_clperbuck = CLUST_BUCK_SZ / MCLBYTES;
|
|
|
|
/*
|
|
* Allocate and initialize PCPU containers.
|
|
*/
|
|
for (i = 0; i < NCPU; i++) {
|
|
if (CPU_ABSENT(i)) {
|
|
mb_statpcpu[i].mb_active = 0;
|
|
continue;
|
|
}
|
|
|
|
mb_list_mbuf.ml_cntlst[i] = malloc(sizeof(struct mb_pcpu_list),
|
|
M_MBUF, M_NOWAIT);
|
|
mb_list_clust.ml_cntlst[i] = malloc(sizeof(struct mb_pcpu_list),
|
|
M_MBUF, M_NOWAIT);
|
|
if ((mb_list_mbuf.ml_cntlst[i] == NULL) ||
|
|
(mb_list_clust.ml_cntlst[i] == NULL))
|
|
goto bad;
|
|
|
|
mtx_init(&mbuf_pcpu[i], "mbuf PCPU list lock", NULL, 0);
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_lock =
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_lock = &mbuf_pcpu[i];
|
|
|
|
mb_statpcpu[i].mb_active = 1;
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_numowner =
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_numowner = i;
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_starved =
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_starved = 0;
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_objcount =
|
|
&(mb_statpcpu[i].mb_mbfree);
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_objcount =
|
|
&(mb_statpcpu[i].mb_clfree);
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_numbucks =
|
|
&(mb_statpcpu[i].mb_mbbucks);
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_numbucks =
|
|
&(mb_statpcpu[i].mb_clbucks);
|
|
mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_types =
|
|
&(mb_statpcpu[i].mb_mbtypes[0]);
|
|
mb_list_clust.ml_cntlst[i]->mb_cont.mc_types = NULL;
|
|
|
|
SLIST_INIT(&(mb_list_mbuf.ml_cntlst[i]->mb_cont.mc_bhead));
|
|
SLIST_INIT(&(mb_list_clust.ml_cntlst[i]->mb_cont.mc_bhead));
|
|
|
|
/*
|
|
* Perform initial allocations.
|
|
*/
|
|
pcpu_cnt = MB_GET_PCPU_LIST_NUM(&mb_list_mbuf, i);
|
|
MB_LOCK_CONT(pcpu_cnt);
|
|
for (j = 0; j < NMB_MBUF_INIT; j++) {
|
|
if (mb_pop_cont(&mb_list_mbuf, M_DONTWAIT, pcpu_cnt)
|
|
== NULL)
|
|
goto bad;
|
|
}
|
|
MB_UNLOCK_CONT(pcpu_cnt);
|
|
|
|
pcpu_cnt = MB_GET_PCPU_LIST_NUM(&mb_list_clust, i);
|
|
MB_LOCK_CONT(pcpu_cnt);
|
|
for (j = 0; j < NMB_CLUST_INIT; j++) {
|
|
if (mb_pop_cont(&mb_list_clust, M_DONTWAIT, pcpu_cnt)
|
|
== NULL)
|
|
goto bad;
|
|
}
|
|
MB_UNLOCK_CONT(pcpu_cnt);
|
|
}
|
|
|
|
return;
|
|
bad:
|
|
panic("mbuf_init(): failed to initialize mbuf subsystem!");
|
|
}
|
|
|
|
/*
|
|
* Populate a given mbuf PCPU container with a bucket full of fresh new
|
|
* buffers. Return a pointer to the new bucket (already in the container if
|
|
* successful), or return NULL on failure.
|
|
*
|
|
* LOCKING NOTES:
|
|
* PCPU container lock must be held when this is called.
|
|
* The lock is dropped here so that we can cleanly call the underlying VM
|
|
* code. If we fail, we return with no locks held. If we succeed (i.e., return
|
|
* non-NULL), we return with the PCPU lock held, ready for allocation from
|
|
* the returned bucket.
|
|
*/
|
|
static struct mb_bucket *
|
|
mb_pop_cont(struct mb_lstmngr *mb_list, int how, struct mb_pcpu_list *cnt_lst)
|
|
{
|
|
struct mb_bucket *bucket;
|
|
caddr_t p;
|
|
int i;
|
|
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
/*
|
|
* If our object's (finite) map is starved now (i.e., no more address
|
|
* space), bail out now.
|
|
*/
|
|
if (mb_list->ml_mapfull)
|
|
return (NULL);
|
|
|
|
bucket = malloc(sizeof(struct mb_bucket) +
|
|
mb_list->ml_objbucks * sizeof(void *), M_MBUF, MBTOM(how));
|
|
if (bucket == NULL)
|
|
return (NULL);
|
|
|
|
p = (caddr_t)kmem_malloc(mb_list->ml_map, mb_list->ml_objsize *
|
|
mb_list->ml_objbucks, MBTOM(how));
|
|
if (p == NULL) {
|
|
free(bucket, M_MBUF);
|
|
if (how == M_TRYWAIT)
|
|
mb_list->ml_mapfull = 1;
|
|
return (NULL);
|
|
}
|
|
|
|
bucket->mb_numfree = 0;
|
|
mb_list->ml_btable[MB_BUCKET_INDX(p, mb_list)] = bucket;
|
|
for (i = 0; i < mb_list->ml_objbucks; i++) {
|
|
bucket->mb_free[i] = p;
|
|
bucket->mb_numfree++;
|
|
p += mb_list->ml_objsize;
|
|
}
|
|
|
|
MB_LOCK_CONT(cnt_lst);
|
|
bucket->mb_owner = cnt_lst->mb_cont.mc_numowner;
|
|
SLIST_INSERT_HEAD(&(cnt_lst->mb_cont.mc_bhead), bucket, mb_blist);
|
|
(*(cnt_lst->mb_cont.mc_numbucks))++;
|
|
*(cnt_lst->mb_cont.mc_objcount) += bucket->mb_numfree;
|
|
|
|
return (bucket);
|
|
}
|
|
|
|
/*
|
|
* Allocate a network buffer.
|
|
* The general case is very easy. Complications only arise if our PCPU
|
|
* container is empty. Things get worse if the PCPU container is empty,
|
|
* the general container is empty, and we've run out of address space
|
|
* in our map; then we try to block if we're willing to (M_TRYWAIT).
|
|
*/
|
|
static __inline
|
|
void *
|
|
mb_alloc(struct mb_lstmngr *mb_list, int how, short type, short persist,
|
|
int *pers_list)
|
|
{
|
|
static int last_report;
|
|
struct mb_pcpu_list *cnt_lst;
|
|
struct mb_bucket *bucket;
|
|
void *m;
|
|
|
|
#ifdef INVARIANTS
|
|
int flags;
|
|
|
|
flags = how & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT);
|
|
if (flags != M_DONTWAIT && flags != M_TRYWAIT) {
|
|
static struct timeval lasterr;
|
|
static int curerr;
|
|
if (ppsratecheck(&lasterr, &curerr, 1)) {
|
|
printf("Bad mbuf alloc flags: %x\n", flags);
|
|
backtrace();
|
|
how = M_TRYWAIT;
|
|
}
|
|
}
|
|
if ((flags & M_DONTWAIT) == 0)
|
|
GIANT_REQUIRED;
|
|
#endif
|
|
|
|
m = NULL;
|
|
if ((persist & MBP_PERSISTENT) != 0) {
|
|
/*
|
|
* If we're a "persistent" call, then the per-CPU #(pers_list)
|
|
* cache lock is already held, and we just need to refer to
|
|
* the correct cache descriptor.
|
|
*/
|
|
cnt_lst = MB_GET_PCPU_LIST_NUM(mb_list, *pers_list);
|
|
} else {
|
|
cnt_lst = MB_GET_PCPU_LIST(mb_list);
|
|
MB_LOCK_CONT(cnt_lst);
|
|
}
|
|
|
|
if ((bucket = SLIST_FIRST(&(cnt_lst->mb_cont.mc_bhead))) != NULL) {
|
|
/*
|
|
* This is the easy allocation case. We just grab an object
|
|
* from a bucket in the PCPU container. At worst, we
|
|
* have just emptied the bucket and so we remove it
|
|
* from the container.
|
|
*/
|
|
MB_GET_OBJECT(m, bucket, cnt_lst);
|
|
MB_MBTYPES_INC(cnt_lst, type, 1);
|
|
|
|
/* If asked to persist, do not drop the lock. */
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
else
|
|
*pers_list = cnt_lst->mb_cont.mc_numowner;
|
|
} else {
|
|
struct mb_gen_list *gen_list;
|
|
|
|
/*
|
|
* This is the less-common more difficult case. We must
|
|
* first verify if the general list has anything for us
|
|
* and if that also fails, we must allocate a page from
|
|
* the map and create a new bucket to place in our PCPU
|
|
* container (already locked). If the map is starved then
|
|
* we're really in for trouble, as we have to wait on
|
|
* the general container's condition variable.
|
|
*/
|
|
gen_list = MB_GET_GEN_LIST(mb_list);
|
|
MB_LOCK_CONT(gen_list);
|
|
|
|
if ((bucket = SLIST_FIRST(&(gen_list->mb_cont.mc_bhead)))
|
|
!= NULL) {
|
|
/*
|
|
* Give ownership of the bucket to our CPU's
|
|
* container, but only actually put the bucket
|
|
* in the container if it doesn't become free
|
|
* upon removing an mbuf from it.
|
|
*/
|
|
SLIST_REMOVE_HEAD(&(gen_list->mb_cont.mc_bhead),
|
|
mb_blist);
|
|
bucket->mb_owner = cnt_lst->mb_cont.mc_numowner;
|
|
(*(gen_list->mb_cont.mc_numbucks))--;
|
|
(*(cnt_lst->mb_cont.mc_numbucks))++;
|
|
*(gen_list->mb_cont.mc_objcount) -= bucket->mb_numfree;
|
|
bucket->mb_numfree--;
|
|
m = bucket->mb_free[(bucket->mb_numfree)];
|
|
if (bucket->mb_numfree == 0) {
|
|
SLIST_NEXT(bucket, mb_blist) = NULL;
|
|
bucket->mb_owner |= MB_BUCKET_FREE;
|
|
} else {
|
|
SLIST_INSERT_HEAD(&(cnt_lst->mb_cont.mc_bhead),
|
|
bucket, mb_blist);
|
|
*(cnt_lst->mb_cont.mc_objcount) +=
|
|
bucket->mb_numfree;
|
|
}
|
|
MB_UNLOCK_CONT(gen_list);
|
|
MB_MBTYPES_INC(cnt_lst, type, 1);
|
|
|
|
/* If asked to persist, do not drop the lock. */
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
else
|
|
*pers_list = cnt_lst->mb_cont.mc_numowner;
|
|
} else {
|
|
/*
|
|
* We'll have to allocate a new page.
|
|
*/
|
|
MB_UNLOCK_CONT(gen_list);
|
|
bucket = mb_pop_cont(mb_list, how, cnt_lst);
|
|
if (bucket != NULL) {
|
|
MB_GET_OBJECT(m, bucket, cnt_lst);
|
|
MB_MBTYPES_INC(cnt_lst, type, 1);
|
|
|
|
/* If asked to persist, do not drop the lock. */
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
else
|
|
*pers_list=cnt_lst->mb_cont.mc_numowner;
|
|
} else {
|
|
if (how == M_TRYWAIT) {
|
|
/*
|
|
* Absolute worst-case scenario.
|
|
* We block if we're willing to, but
|
|
* only after trying to steal from
|
|
* other lists.
|
|
*/
|
|
m = mb_alloc_wait(mb_list, type);
|
|
} else {
|
|
/* XXX: No consistency. */
|
|
mbstat.m_drops++;
|
|
|
|
if (ticks < last_report ||
|
|
(ticks - last_report) >= hz) {
|
|
last_report = ticks;
|
|
printf(
|
|
"All mbufs or mbuf clusters exhausted, please see tuning(7).\n");
|
|
}
|
|
|
|
}
|
|
if (m != NULL && (persist & MBP_PERSIST) != 0) {
|
|
cnt_lst = MB_GET_PCPU_LIST(mb_list);
|
|
MB_LOCK_CONT(cnt_lst);
|
|
*pers_list=cnt_lst->mb_cont.mc_numowner;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return (m);
|
|
}
|
|
|
|
/*
|
|
* This is the worst-case scenario called only if we're allocating with
|
|
* M_TRYWAIT. We first drain all the protocols, then try to find an mbuf
|
|
* by looking in every PCPU container. If we're still unsuccesful, we
|
|
* try the general container one last time and possibly block on our
|
|
* starved cv.
|
|
*/
|
|
static void *
|
|
mb_alloc_wait(struct mb_lstmngr *mb_list, short type)
|
|
{
|
|
struct mb_pcpu_list *cnt_lst;
|
|
struct mb_gen_list *gen_list;
|
|
struct mb_bucket *bucket;
|
|
void *m;
|
|
int i, cv_ret;
|
|
|
|
/*
|
|
* Try to reclaim mbuf-related objects (mbufs, clusters).
|
|
*/
|
|
mb_reclaim();
|
|
|
|
/*
|
|
* Cycle all the PCPU containers. Increment starved counts if found
|
|
* empty.
|
|
*/
|
|
for (i = 0; i < NCPU; i++) {
|
|
if (CPU_ABSENT(i))
|
|
continue;
|
|
cnt_lst = MB_GET_PCPU_LIST_NUM(mb_list, i);
|
|
MB_LOCK_CONT(cnt_lst);
|
|
|
|
/*
|
|
* If container is non-empty, get a single object from it.
|
|
* If empty, increment starved count.
|
|
*/
|
|
if ((bucket = SLIST_FIRST(&(cnt_lst->mb_cont.mc_bhead))) !=
|
|
NULL) {
|
|
MB_GET_OBJECT(m, bucket, cnt_lst);
|
|
MB_MBTYPES_INC(cnt_lst, type, 1);
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
mbstat.m_wait++; /* XXX: No consistency. */
|
|
return (m);
|
|
} else
|
|
cnt_lst->mb_cont.mc_starved++;
|
|
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
}
|
|
|
|
/*
|
|
* We're still here, so that means it's time to get the general
|
|
* container lock, check it one more time (now that mb_reclaim()
|
|
* has been called) and if we still get nothing, block on the cv.
|
|
*/
|
|
gen_list = MB_GET_GEN_LIST(mb_list);
|
|
MB_LOCK_CONT(gen_list);
|
|
if ((bucket = SLIST_FIRST(&(gen_list->mb_cont.mc_bhead))) != NULL) {
|
|
MB_GET_OBJECT(m, bucket, gen_list);
|
|
MB_MBTYPES_INC(gen_list, type, 1);
|
|
MB_UNLOCK_CONT(gen_list);
|
|
mbstat.m_wait++; /* XXX: No consistency. */
|
|
return (m);
|
|
}
|
|
|
|
gen_list->mb_cont.mc_starved++;
|
|
cv_ret = cv_timedwait(&(gen_list->mgl_mstarved),
|
|
gen_list->mb_cont.mc_lock, mbuf_wait);
|
|
gen_list->mb_cont.mc_starved--;
|
|
|
|
if ((cv_ret == 0) &&
|
|
((bucket = SLIST_FIRST(&(gen_list->mb_cont.mc_bhead))) != NULL)) {
|
|
MB_GET_OBJECT(m, bucket, gen_list);
|
|
MB_MBTYPES_INC(gen_list, type, 1);
|
|
mbstat.m_wait++; /* XXX: No consistency. */
|
|
} else {
|
|
mbstat.m_drops++; /* XXX: No consistency. */
|
|
m = NULL;
|
|
}
|
|
|
|
MB_UNLOCK_CONT(gen_list);
|
|
|
|
return (m);
|
|
}
|
|
|
|
/*-
|
|
* Free an object to its rightful container.
|
|
* In the very general case, this operation is really very easy.
|
|
* Complications arise primarily if:
|
|
* (a) We've hit the high limit on number of free objects allowed in
|
|
* our PCPU container.
|
|
* (b) We're in a critical situation where our container has been
|
|
* marked 'starved' and we need to issue wakeups on the starved
|
|
* condition variable.
|
|
* (c) Minor (odd) cases: our bucket has migrated while we were
|
|
* waiting for the lock; our bucket is in the general container;
|
|
* our bucket is empty.
|
|
*/
|
|
static __inline
|
|
void
|
|
mb_free(struct mb_lstmngr *mb_list, void *m, short type, short persist,
|
|
int *pers_list)
|
|
{
|
|
struct mb_pcpu_list *cnt_lst;
|
|
struct mb_gen_list *gen_list;
|
|
struct mb_bucket *bucket;
|
|
u_int owner;
|
|
|
|
bucket = mb_list->ml_btable[MB_BUCKET_INDX(m, mb_list)];
|
|
|
|
/*
|
|
* Make sure that if after we lock the bucket's present container the
|
|
* bucket has migrated, that we drop the lock and get the new one.
|
|
*/
|
|
retry_lock:
|
|
owner = bucket->mb_owner & ~MB_BUCKET_FREE;
|
|
switch (owner) {
|
|
case MB_GENLIST_OWNER:
|
|
gen_list = MB_GET_GEN_LIST(mb_list);
|
|
if (((persist & MBP_PERSISTENT) != 0) && (*pers_list >= 0)) {
|
|
if (*pers_list != MB_GENLIST_OWNER) {
|
|
cnt_lst = MB_GET_PCPU_LIST_NUM(mb_list,
|
|
*pers_list);
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
MB_LOCK_CONT(gen_list);
|
|
}
|
|
} else {
|
|
MB_LOCK_CONT(gen_list);
|
|
}
|
|
if (owner != (bucket->mb_owner & ~MB_BUCKET_FREE)) {
|
|
MB_UNLOCK_CONT(gen_list);
|
|
*pers_list = -1;
|
|
goto retry_lock;
|
|
}
|
|
|
|
/*
|
|
* If we're intended for the general container, this is
|
|
* real easy: no migrating required. The only `bogon'
|
|
* is that we're now contending with all the threads
|
|
* dealing with the general list, but this is expected.
|
|
*/
|
|
MB_PUT_OBJECT(m, bucket, gen_list);
|
|
MB_MBTYPES_DEC(gen_list, type, 1);
|
|
if (bucket->mb_owner & MB_BUCKET_FREE) {
|
|
SLIST_INSERT_HEAD(&(gen_list->mb_cont.mc_bhead),
|
|
bucket, mb_blist);
|
|
bucket->mb_owner = MB_GENLIST_OWNER;
|
|
}
|
|
if (gen_list->mb_cont.mc_starved > 0)
|
|
cv_signal(&(gen_list->mgl_mstarved));
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(gen_list);
|
|
else
|
|
*pers_list = MB_GENLIST_OWNER;
|
|
break;
|
|
|
|
default:
|
|
cnt_lst = MB_GET_PCPU_LIST_NUM(mb_list, owner);
|
|
if (((persist & MBP_PERSISTENT) != 0) && (*pers_list >= 0)) {
|
|
if (*pers_list == MB_GENLIST_OWNER) {
|
|
gen_list = MB_GET_GEN_LIST(mb_list);
|
|
MB_UNLOCK_CONT(gen_list);
|
|
MB_LOCK_CONT(cnt_lst);
|
|
} else {
|
|
cnt_lst = MB_GET_PCPU_LIST_NUM(mb_list,
|
|
*pers_list);
|
|
owner = *pers_list;
|
|
}
|
|
} else {
|
|
MB_LOCK_CONT(cnt_lst);
|
|
}
|
|
if (owner != (bucket->mb_owner & ~MB_BUCKET_FREE)) {
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
*pers_list = -1;
|
|
goto retry_lock;
|
|
}
|
|
|
|
MB_PUT_OBJECT(m, bucket, cnt_lst);
|
|
MB_MBTYPES_DEC(cnt_lst, type, 1);
|
|
if ((*(cnt_lst->mb_cont.mc_objcount) > *(mb_list->ml_wmhigh)) ||
|
|
(cnt_lst->mb_cont.mc_starved > 0)) {
|
|
/*
|
|
* We've hit the high limit of allowed numbers of mbufs
|
|
* on this PCPU list or we've been flagged that we need
|
|
* to transfer a bucket over to the general cache.
|
|
* We must now migrate a bucket over to the general
|
|
* container.
|
|
*/
|
|
gen_list = MB_GET_GEN_LIST(mb_list);
|
|
MB_LOCK_CONT(gen_list);
|
|
if ((bucket->mb_owner & MB_BUCKET_FREE) == 0) {
|
|
bucket =
|
|
SLIST_FIRST(&(cnt_lst->mb_cont.mc_bhead));
|
|
SLIST_REMOVE_HEAD(&(cnt_lst->mb_cont.mc_bhead),
|
|
mb_blist);
|
|
}
|
|
SLIST_INSERT_HEAD(&(gen_list->mb_cont.mc_bhead),
|
|
bucket, mb_blist);
|
|
bucket->mb_owner = MB_GENLIST_OWNER;
|
|
*(cnt_lst->mb_cont.mc_objcount) -= bucket->mb_numfree;
|
|
*(gen_list->mb_cont.mc_objcount) += bucket->mb_numfree;
|
|
(*(cnt_lst->mb_cont.mc_numbucks))--;
|
|
(*(gen_list->mb_cont.mc_numbucks))++;
|
|
|
|
/*
|
|
* While we're at it, transfer some of the mbtypes
|
|
* "count load" onto the general list's mbtypes
|
|
* array, seeing as how we're moving the bucket
|
|
* there now, meaning that the freeing of objects
|
|
* there will now decrement the _general list's_
|
|
* mbtypes counters, and no longer our PCPU list's
|
|
* mbtypes counters. We do this for the type presently
|
|
* being freed in an effort to keep the mbtypes
|
|
* counters approximately balanced across all lists.
|
|
*/
|
|
MB_MBTYPES_DEC(cnt_lst, type,
|
|
mb_list->ml_objbucks - bucket->mb_numfree);
|
|
MB_MBTYPES_INC(gen_list, type,
|
|
mb_list->ml_objbucks - bucket->mb_numfree);
|
|
|
|
if (cnt_lst->mb_cont.mc_starved > 0) {
|
|
/*
|
|
* Determine whether or not to keep
|
|
* transferring buckets to the general list
|
|
* or whether we've transferred enough already.
|
|
* The thread that is blocked may end up waking
|
|
* up in the meantime, but transferring an
|
|
* extra bucket in a constrained situation
|
|
* is not so bad, as we're likely to need
|
|
* it soon anyway.
|
|
*/
|
|
if (gen_list->mb_cont.mc_starved > 0) {
|
|
cnt_lst->mb_cont.mc_starved--;
|
|
cv_signal(&(gen_list->mgl_mstarved));
|
|
} else
|
|
cnt_lst->mb_cont.mc_starved = 0;
|
|
}
|
|
MB_UNLOCK_CONT(gen_list);
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
else
|
|
*pers_list = owner;
|
|
break;
|
|
}
|
|
|
|
if (bucket->mb_owner & MB_BUCKET_FREE) {
|
|
SLIST_INSERT_HEAD(&(cnt_lst->mb_cont.mc_bhead),
|
|
bucket, mb_blist);
|
|
bucket->mb_owner = cnt_lst->mb_cont.mc_numowner;
|
|
}
|
|
|
|
if ((persist & MBP_PERSIST) == 0)
|
|
MB_UNLOCK_CONT(cnt_lst);
|
|
else
|
|
*pers_list = owner;
|
|
break;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Drain protocols in hopes to free up some resources.
|
|
*
|
|
* LOCKING NOTES:
|
|
* No locks should be held when this is called. The drain routines have to
|
|
* presently acquire some locks which raises the possibility of lock order
|
|
* violation if we're holding any mutex if that mutex is acquired in reverse
|
|
* order relative to one of the locks in the drain routines.
|
|
*/
|
|
static void
|
|
mb_reclaim(void)
|
|
{
|
|
struct domain *dp;
|
|
struct protosw *pr;
|
|
|
|
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
|
|
"mb_reclaim()");
|
|
|
|
mbstat.m_drain++; /* XXX: No consistency. */
|
|
|
|
for (dp = domains; dp != NULL; dp = dp->dom_next)
|
|
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
|
|
if (pr->pr_drain != NULL)
|
|
(*pr->pr_drain)();
|
|
}
|
|
|
|
/******************************************************************************
|
|
* Internal setup macros.
|
|
*/
|
|
|
|
#define _mb_setup(m, type) do { \
|
|
(m)->m_type = (type); \
|
|
(m)->m_next = NULL; \
|
|
(m)->m_nextpkt = NULL; \
|
|
(m)->m_data = (m)->m_dat; \
|
|
(m)->m_flags = 0; \
|
|
} while (0)
|
|
|
|
#define _mbhdr_setup(m, type) do { \
|
|
(m)->m_type = (type); \
|
|
(m)->m_next = NULL; \
|
|
(m)->m_nextpkt = NULL; \
|
|
(m)->m_data = (m)->m_pktdat; \
|
|
(m)->m_flags = M_PKTHDR; \
|
|
(m)->m_pkthdr.rcvif = NULL; \
|
|
(m)->m_pkthdr.csum_flags = 0; \
|
|
SLIST_INIT(&(m)->m_pkthdr.tags); \
|
|
} while (0)
|
|
|
|
#define _mcl_setup(m) do { \
|
|
(m)->m_data = (m)->m_ext.ext_buf; \
|
|
(m)->m_flags |= M_EXT; \
|
|
(m)->m_ext.ext_free = NULL; \
|
|
(m)->m_ext.ext_args = NULL; \
|
|
(m)->m_ext.ext_size = MCLBYTES; \
|
|
(m)->m_ext.ext_type = EXT_CLUSTER; \
|
|
} while (0)
|
|
|
|
#define _mext_init_ref(m, ref) do { \
|
|
(m)->m_ext.ref_cnt = ((ref) == NULL) ? \
|
|
malloc(sizeof(u_int), M_MBUF, M_NOWAIT) : (u_int *)(ref); \
|
|
if ((m)->m_ext.ref_cnt != NULL) { \
|
|
*((m)->m_ext.ref_cnt) = 0; \
|
|
MEXT_ADD_REF((m)); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define cl2ref(cl) \
|
|
(((uintptr_t)(cl) - (uintptr_t)mb_list_clust.ml_mapbase) >> MCLSHIFT)
|
|
|
|
#define _mext_dealloc_ref(m) \
|
|
if ((m)->m_ext.ext_type != EXT_EXTREF) \
|
|
free((m)->m_ext.ref_cnt, M_MBUF)
|
|
|
|
/******************************************************************************
|
|
* Internal routines.
|
|
*
|
|
* Because mb_alloc() and mb_free() are inlines (to keep the common
|
|
* cases down to a maximum of one function call), below are a few
|
|
* routines used only internally for the sole purpose of making certain
|
|
* functions smaller.
|
|
*
|
|
* - _mext_free(): frees associated storage when the ref. count is
|
|
* exactly one and we're freeing.
|
|
*
|
|
* - _mgetm_internal(): common "persistent-lock" routine that allocates
|
|
* an mbuf and a cluster in one shot, but where the lock is already
|
|
* held coming in (which is what makes it different from the exported
|
|
* m_getcl()). The lock is dropped when done. This is used by m_getm()
|
|
* and, therefore, is very m_getm()-specific.
|
|
*/
|
|
static struct mbuf *_mgetm_internal(int, short, short, int);
|
|
|
|
void
|
|
_mext_free(struct mbuf *mb)
|
|
{
|
|
|
|
if (mb->m_ext.ext_type == EXT_CLUSTER) {
|
|
mb_free(&mb_list_clust, (caddr_t)mb->m_ext.ext_buf, MT_NOTMBUF,
|
|
0, NULL);
|
|
} else {
|
|
(*(mb->m_ext.ext_free))(mb->m_ext.ext_buf, mb->m_ext.ext_args);
|
|
_mext_dealloc_ref(mb);
|
|
}
|
|
}
|
|
|
|
static struct mbuf *
|
|
_mgetm_internal(int how, short type, short persist, int cchnum)
|
|
{
|
|
struct mbuf *mb;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type, persist,&cchnum);
|
|
if (mb == NULL)
|
|
return NULL;
|
|
_mb_setup(mb, type);
|
|
|
|
if ((persist & MBP_PERSIST) != 0) {
|
|
mb->m_ext.ext_buf = (caddr_t)mb_alloc(&mb_list_clust,
|
|
how, MT_NOTMBUF, MBP_PERSISTENT, &cchnum);
|
|
if (mb->m_ext.ext_buf == NULL) {
|
|
(void)m_free(mb);
|
|
mb = NULL;
|
|
}
|
|
_mcl_setup(mb);
|
|
_mext_init_ref(mb, &cl_refcntmap[cl2ref(mb->m_ext.ext_buf)]);
|
|
}
|
|
return (mb);
|
|
}
|
|
|
|
/******************************************************************************
|
|
* Exported buffer allocation and de-allocation routines.
|
|
*/
|
|
|
|
/*
|
|
* Allocate and return a single (normal) mbuf. NULL is returned on failure.
|
|
*
|
|
* Arguments:
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
*/
|
|
struct mbuf *
|
|
m_get(int how, short type)
|
|
{
|
|
struct mbuf *mb;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type, 0, NULL);
|
|
if (mb != NULL)
|
|
_mb_setup(mb, type);
|
|
return (mb);
|
|
}
|
|
|
|
/*
|
|
* Allocate a given length worth of mbufs and/or clusters (whatever fits
|
|
* best) and return a pointer to the top of the allocated chain. If an
|
|
* existing mbuf chain is provided, then we will append the new chain
|
|
* to the existing one but still return the top of the newly allocated
|
|
* chain. NULL is returned on failure, in which case the [optional]
|
|
* provided chain is left untouched, and any memory already allocated
|
|
* is freed.
|
|
*
|
|
* Arguments:
|
|
* - m: existing chain to which to append new chain (optional).
|
|
* - len: total length of data to append, either in mbufs or clusters
|
|
* (we allocate whatever combination yields the best fit).
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
*/
|
|
struct mbuf *
|
|
m_getm(struct mbuf *m, int len, int how, short type)
|
|
{
|
|
struct mbuf *mb, *top, *cur, *mtail;
|
|
int num, rem, cchnum;
|
|
short persist;
|
|
int i;
|
|
|
|
KASSERT(len >= 0, ("m_getm(): len is < 0"));
|
|
|
|
/* If m != NULL, we will append to the end of that chain. */
|
|
if (m != NULL)
|
|
for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
|
|
else
|
|
mtail = NULL;
|
|
|
|
/*
|
|
* In the best-case scenario (which should be the common case
|
|
* unless we're in a starvation situation), we will be able to
|
|
* go through the allocation of all the desired mbufs and clusters
|
|
* here without dropping our per-CPU cache lock in between.
|
|
*/
|
|
num = len / MCLBYTES;
|
|
rem = len % MCLBYTES;
|
|
persist = 0;
|
|
cchnum = -1;
|
|
top = cur = NULL;
|
|
for (i = 0; i < num; i++) {
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type,
|
|
MBP_PERSIST | persist, &cchnum);
|
|
if (mb == NULL)
|
|
goto failed;
|
|
_mb_setup(mb, type);
|
|
mb->m_len = 0;
|
|
|
|
persist = (i != (num - 1) || rem > 0) ? MBP_PERSIST : 0;
|
|
mb->m_ext.ext_buf = (caddr_t)mb_alloc(&mb_list_clust,
|
|
how, MT_NOTMBUF, persist | MBP_PERSISTENT, &cchnum);
|
|
if (mb->m_ext.ext_buf == NULL) {
|
|
(void)m_free(mb);
|
|
goto failed;
|
|
}
|
|
_mcl_setup(mb);
|
|
_mext_init_ref(mb, &cl_refcntmap[cl2ref(mb->m_ext.ext_buf)]);
|
|
persist = MBP_PERSISTENT;
|
|
|
|
if (cur == NULL)
|
|
top = cur = mb;
|
|
else
|
|
cur = (cur->m_next = mb);
|
|
}
|
|
if (rem > 0) {
|
|
if (cchnum >= 0) {
|
|
persist = MBP_PERSISTENT;
|
|
persist |= (rem > MINCLSIZE) ? MBP_PERSIST : 0;
|
|
mb = _mgetm_internal(how, type, persist, cchnum);
|
|
if (mb == NULL)
|
|
goto failed;
|
|
} else if (rem > MINCLSIZE) {
|
|
mb = m_getcl(how, type, 0);
|
|
} else {
|
|
mb = m_get(how, type);
|
|
}
|
|
if (mb != NULL) {
|
|
mb->m_len = 0;
|
|
if (cur == NULL)
|
|
top = mb;
|
|
else
|
|
cur->m_next = mb;
|
|
} else
|
|
goto failed;
|
|
}
|
|
|
|
if (mtail != NULL)
|
|
mtail->m_next = top;
|
|
return top;
|
|
failed:
|
|
if (top != NULL)
|
|
m_freem(top);
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Allocate and return a single M_PKTHDR mbuf. NULL is returned on failure.
|
|
*
|
|
* Arguments:
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
*/
|
|
struct mbuf *
|
|
m_gethdr(int how, short type)
|
|
{
|
|
struct mbuf *mb;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type, 0, NULL);
|
|
if (mb != NULL) {
|
|
_mbhdr_setup(mb, type);
|
|
#ifdef MAC
|
|
if (mac_init_mbuf(mb, MBTOM(how)) != 0) {
|
|
m_free(mb);
|
|
return (NULL);
|
|
}
|
|
#endif
|
|
}
|
|
return (mb);
|
|
}
|
|
|
|
/*
|
|
* Allocate and return a single (normal) pre-zero'd mbuf. NULL is
|
|
* returned on failure.
|
|
*
|
|
* Arguments:
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
*/
|
|
struct mbuf *
|
|
m_get_clrd(int how, short type)
|
|
{
|
|
struct mbuf *mb;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type, 0, NULL);
|
|
if (mb != NULL) {
|
|
_mb_setup(mb, type);
|
|
bzero(mtod(mb, caddr_t), MLEN);
|
|
}
|
|
return (mb);
|
|
}
|
|
|
|
/*
|
|
* Allocate and return a single M_PKTHDR pre-zero'd mbuf. NULL is
|
|
* returned on failure.
|
|
*
|
|
* Arguments:
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
*/
|
|
struct mbuf *
|
|
m_gethdr_clrd(int how, short type)
|
|
{
|
|
struct mbuf *mb;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type, 0, NULL);
|
|
if (mb != NULL) {
|
|
_mbhdr_setup(mb, type);
|
|
#ifdef MAC
|
|
if (mac_init_mbuf(mb, MBTOM(how)) != 0) {
|
|
m_free(mb);
|
|
return (NULL);
|
|
}
|
|
#endif
|
|
bzero(mtod(mb, caddr_t), MHLEN);
|
|
}
|
|
return (mb);
|
|
}
|
|
|
|
/*
|
|
* Free a single mbuf and any associated storage that it may have attached
|
|
* to it. The associated storage may not be immediately freed if its
|
|
* reference count is above 1. Returns the next mbuf in the chain following
|
|
* the mbuf being freed.
|
|
*
|
|
* Arguments:
|
|
* - mb: the mbuf to free.
|
|
*/
|
|
struct mbuf *
|
|
m_free(struct mbuf *mb)
|
|
{
|
|
struct mbuf *nb;
|
|
int cchnum;
|
|
short persist = 0;
|
|
|
|
#ifdef INVARIANTS
|
|
if (mb->m_flags & M_FREELIST)
|
|
panic("m_free detected a mbuf double-free");
|
|
mb->m_flags |= M_FREELIST;
|
|
#endif
|
|
if ((mb->m_flags & M_PKTHDR) != 0)
|
|
m_tag_delete_chain(mb, NULL);
|
|
nb = mb->m_next;
|
|
if ((mb->m_flags & M_EXT) != 0) {
|
|
MEXT_REM_REF(mb);
|
|
if (atomic_cmpset_int(mb->m_ext.ref_cnt, 0, 1)) {
|
|
if (mb->m_ext.ext_type == EXT_CLUSTER) {
|
|
mb_free(&mb_list_clust,
|
|
(caddr_t)mb->m_ext.ext_buf, MT_NOTMBUF,
|
|
MBP_PERSIST, &cchnum);
|
|
persist = MBP_PERSISTENT;
|
|
} else {
|
|
(*(mb->m_ext.ext_free))(mb->m_ext.ext_buf,
|
|
mb->m_ext.ext_args);
|
|
_mext_dealloc_ref(mb);
|
|
persist = 0;
|
|
}
|
|
}
|
|
}
|
|
mb_free(&mb_list_mbuf, mb, mb->m_type, persist, &cchnum);
|
|
return (nb);
|
|
}
|
|
|
|
/*
|
|
* Free an entire chain of mbufs and associated external buffers, if
|
|
* applicable. Right now, we only optimize a little so that the cache
|
|
* lock may be held across a single mbuf+cluster free. Hopefully,
|
|
* we'll eventually be holding the lock across more than merely two
|
|
* consecutive frees but right now this is hard to implement because of
|
|
* things like _mext_dealloc_ref (may do a free()) and atomic ops in the
|
|
* loop.
|
|
*
|
|
* - mb: the mbuf chain to free.
|
|
*/
|
|
void
|
|
m_freem(struct mbuf *mb)
|
|
{
|
|
|
|
while (mb != NULL)
|
|
mb = m_free(mb);
|
|
}
|
|
|
|
/*
|
|
* Fetch an mbuf with a cluster attached to it. If one of the
|
|
* allocations fails, the entire allocation fails. This routine is
|
|
* the preferred way of fetching both the mbuf and cluster together,
|
|
* as it avoids having to unlock/relock between allocations. Returns
|
|
* NULL on failure.
|
|
*
|
|
* Arguments:
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
* - type: the type of the mbuf being allocated.
|
|
* - flags: any flags to pass to the mbuf being allocated; if this includes
|
|
* the M_PKTHDR bit, then the mbuf is configured as a M_PKTHDR mbuf.
|
|
*/
|
|
struct mbuf *
|
|
m_getcl(int how, short type, int flags)
|
|
{
|
|
struct mbuf *mb;
|
|
int cchnum;
|
|
|
|
mb = (struct mbuf *)mb_alloc(&mb_list_mbuf, how, type,
|
|
MBP_PERSIST, &cchnum);
|
|
if (mb == NULL)
|
|
return NULL;
|
|
mb->m_type = type;
|
|
mb->m_next = NULL;
|
|
mb->m_flags = flags;
|
|
if ((flags & M_PKTHDR) != 0) {
|
|
mb->m_nextpkt = NULL;
|
|
mb->m_pkthdr.rcvif = NULL;
|
|
mb->m_pkthdr.csum_flags = 0;
|
|
SLIST_INIT(&mb->m_pkthdr.tags);
|
|
}
|
|
|
|
mb->m_ext.ext_buf = (caddr_t)mb_alloc(&mb_list_clust, how,
|
|
MT_NOTMBUF, MBP_PERSISTENT, &cchnum);
|
|
if (mb->m_ext.ext_buf == NULL) {
|
|
(void)m_free(mb);
|
|
mb = NULL;
|
|
} else {
|
|
_mcl_setup(mb);
|
|
_mext_init_ref(mb, &cl_refcntmap[cl2ref(mb->m_ext.ext_buf)]);
|
|
}
|
|
#ifdef MAC
|
|
if (flags & M_PKTHDR) {
|
|
if (mac_init_mbuf(mb, MBTOM(how)) != 0) {
|
|
m_free(mb);
|
|
return (NULL);
|
|
}
|
|
}
|
|
#endif
|
|
return (mb);
|
|
}
|
|
|
|
/*
|
|
* Fetch a single mbuf cluster and attach it to an existing mbuf. If
|
|
* successfull, configures the provided mbuf to have mbuf->m_ext.ext_buf
|
|
* pointing to the cluster, and sets the M_EXT bit in the mbuf's flags.
|
|
* The M_EXT bit is not set on failure.
|
|
*
|
|
* Arguments:
|
|
* - mb: the existing mbuf to which to attach the allocated cluster.
|
|
* - how: M_TRYWAIT to try to block for kern.ipc.mbuf_wait number of ticks
|
|
* if really starved for memory. M_DONTWAIT to never block.
|
|
*/
|
|
void
|
|
m_clget(struct mbuf *mb, int how)
|
|
{
|
|
|
|
mb->m_ext.ext_buf= (caddr_t)mb_alloc(&mb_list_clust,how,MT_NOTMBUF,
|
|
0, NULL);
|
|
if (mb->m_ext.ext_buf != NULL) {
|
|
_mcl_setup(mb);
|
|
_mext_init_ref(mb, &cl_refcntmap[cl2ref(mb->m_ext.ext_buf)]);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Configure a provided mbuf to refer to the provided external storage
|
|
* buffer and setup a reference count for said buffer. If the setting
|
|
* up of the reference count fails, the M_EXT bit will not be set. If
|
|
* successfull, the M_EXT bit is set in the mbuf's flags.
|
|
*
|
|
* Arguments:
|
|
* - mb: the existing mbuf to which to attach the provided buffer.
|
|
* - buf: the address of the provided external storage buffer.
|
|
* - size: the size of the provided buffer.
|
|
* - freef: a pointer to a routine that is responsible for freeing the
|
|
* provided external storage buffer.
|
|
* - args: a pointer to an argument structure (of any type) to be passed
|
|
* to the provided freef routine (may be NULL).
|
|
* - flags: any other flags to be passed to the provided mbuf.
|
|
* - type: the type that the external storage buffer should be labeled with.
|
|
*/
|
|
void
|
|
m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
|
|
void (*freef)(void *, void *), void *args, int flags, int type)
|
|
{
|
|
u_int *ref_cnt = NULL;
|
|
|
|
if (type == EXT_CLUSTER)
|
|
ref_cnt = &cl_refcntmap[cl2ref(mb->m_ext.ext_buf)];
|
|
else if (type == EXT_EXTREF)
|
|
ref_cnt = mb->m_ext.ref_cnt;
|
|
_mext_init_ref(mb, ref_cnt);
|
|
if (mb->m_ext.ref_cnt != NULL) {
|
|
mb->m_flags |= (M_EXT | flags);
|
|
mb->m_ext.ext_buf = buf;
|
|
mb->m_data = mb->m_ext.ext_buf;
|
|
mb->m_ext.ext_size = size;
|
|
mb->m_ext.ext_free = freef;
|
|
mb->m_ext.ext_args = args;
|
|
mb->m_ext.ext_type = type;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Change type of provided mbuf. This is a relatively expensive operation
|
|
* (due to the cost of statistics manipulations) and should be avoided, where
|
|
* possible.
|
|
*
|
|
* Arguments:
|
|
* - mb: the provided mbuf for which the type needs to be changed.
|
|
* - new_type: the new type to change the mbuf to.
|
|
*/
|
|
void
|
|
m_chtype(struct mbuf *mb, short new_type)
|
|
{
|
|
struct mb_gen_list *gen_list;
|
|
|
|
gen_list = MB_GET_GEN_LIST(&mb_list_mbuf);
|
|
MB_LOCK_CONT(gen_list);
|
|
MB_MBTYPES_DEC(gen_list, mb->m_type, 1);
|
|
MB_MBTYPES_INC(gen_list, new_type, 1);
|
|
MB_UNLOCK_CONT(gen_list);
|
|
mb->m_type = new_type;
|
|
}
|