diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index a60224957507..fba9474e4581 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -115,6 +115,9 @@ static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); /* This mutex protects the zone list */ static struct mtx uma_mtx; +/* These are the pcpu cache locks */ +static struct mtx uma_pcpu_mtx[MAXCPU]; + /* Linked list of boot time pages */ static LIST_HEAD(,uma_slab) uma_boot_pages = LIST_HEAD_INITIALIZER(&uma_boot_pages); @@ -249,7 +252,7 @@ zone_timeout(uma_zone_t zone) for (cpu = 0; cpu < maxcpu; cpu++) { if (CPU_ABSENT(cpu)) continue; - CPU_LOCK(zone, cpu); + CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; /* Add them up, and reset */ alloc += cache->uc_allocs; @@ -258,7 +261,7 @@ zone_timeout(uma_zone_t zone) free += cache->uc_allocbucket->ub_ptr + 1; if (cache->uc_freebucket) free += cache->uc_freebucket->ub_ptr + 1; - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); } } @@ -514,7 +517,7 @@ cache_drain(uma_zone_t zone) for (cpu = 0; cpu < maxcpu; cpu++) { if (CPU_ABSENT(cpu)) continue; - CPU_LOCK(zone, cpu); + CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; bucket_drain(zone, cache->uc_allocbucket); bucket_drain(zone, cache->uc_freebucket); @@ -543,7 +546,7 @@ cache_drain(uma_zone_t zone) for (cpu = 0; cpu < maxcpu; cpu++) { if (CPU_ABSENT(cpu)) continue; - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); } zone->uz_cachefree = 0; @@ -985,8 +988,6 @@ zone_ctor(void *mem, int size, void *udata) struct uma_zctor_args *arg = udata; uma_zone_t zone = mem; int privlc; - int cplen; - int cpu; bzero(zone, size); zone->uz_name = arg->name; @@ -1033,12 +1034,6 @@ zone_ctor(void *mem, int size, void *udata) else privlc = 0; - /* We do this so that the per cpu lock name is unique for each zone */ - memcpy(zone->uz_lname, "PCPU ", 5); - cplen = min(strlen(zone->uz_name) + 1, LOCKNAME_LEN - 6); - memcpy(zone->uz_lname+5, zone->uz_name, cplen); - zone->uz_lname[LOCKNAME_LEN - 1] = '\0'; - /* * If we're putting the slab header in the actual page we need to * figure out where in each page it goes. This calculates a right @@ -1107,9 +1102,6 @@ zone_ctor(void *mem, int size, void *udata) zone->uz_count = zone->uz_ipers - 1; else zone->uz_count = UMA_BUCKET_SIZE - 1; - - for (cpu = 0; cpu < maxcpu; cpu++) - CPU_LOCK_INIT(zone, cpu, privlc); } /* @@ -1124,10 +1116,8 @@ static void zone_dtor(void *arg, int size, void *udata) { uma_zone_t zone; - int cpu; zone = (uma_zone_t)arg; - ZONE_LOCK(zone); zone->uz_wssize = 0; ZONE_UNLOCK(zone); @@ -1142,10 +1132,6 @@ zone_dtor(void *arg, int size, void *udata) printf("Zone %s was not empty (%d items). Lost %d pages of memory.\n", zone->uz_name, zone->uz_free, zone->uz_pages); - if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) - for (cpu = 0; cpu < maxcpu; cpu++) - CPU_LOCK_FINI(zone, cpu); - ZONE_UNLOCK(zone); if ((zone->uz_flags & UMA_ZFLAG_OFFPAGE) != 0) hash_free(&zone->uz_hash); @@ -1210,6 +1196,10 @@ uma_startup(void *bootmem) /* The initial zone has no Per cpu queues so it's smaller */ zone_ctor(zones, sizeof(struct uma_zone), &args); + /* Initialize the pcpu cache lock set once and for all */ + for (i = 0; i < maxcpu; i++) + CPU_LOCK_INIT(i); + #ifdef UMA_DEBUG printf("Filling boot free list.\n"); #endif @@ -1339,7 +1329,7 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) zalloc_restart: cpu = PCPU_GET(cpuid); - CPU_LOCK(zone, cpu); + CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zalloc_start: @@ -1360,7 +1350,7 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) uma_dbg_alloc(zone, NULL, item); ZONE_UNLOCK(zone); #endif - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); if (zone->uz_ctor) zone->uz_ctor(item, zone->uz_size, udata); if (flags & M_ZERO) @@ -1410,7 +1400,7 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) goto zalloc_start; } /* We are no longer associated with this cpu!!! */ - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); /* Bump up our uz_count so we get here less */ if (zone->uz_count < UMA_BUCKET_SIZE - 1) @@ -1655,7 +1645,7 @@ uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) ZONE_UNLOCK(zone); - if (zone->uz_ctor != NULL) + if (zone->uz_ctor != NULL) zone->uz_ctor(item, zone->uz_size, udata); if (flags & M_ZERO) bzero(item, zone->uz_size); @@ -1693,7 +1683,7 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata) zfree_restart: cpu = PCPU_GET(cpuid); - CPU_LOCK(zone, cpu); + CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zfree_start: @@ -1718,7 +1708,7 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata) uma_dbg_free(zone, NULL, item); ZONE_UNLOCK(zone); #endif - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); return; } else if (cache->uc_allocbucket) { #ifdef UMA_DEBUG_ALLOC @@ -1772,7 +1762,7 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata) goto zfree_start; } /* We're done with this CPU now */ - CPU_UNLOCK(zone, cpu); + CPU_UNLOCK(cpu); /* And the zone.. */ ZONE_UNLOCK(zone); diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h index 8d923368f6d9..78493c766788 100644 --- a/sys/vm/uma_int.h +++ b/sys/vm/uma_int.h @@ -190,7 +190,6 @@ struct uma_bucket { typedef struct uma_bucket * uma_bucket_t; struct uma_cache { - struct mtx uc_lock; /* Spin lock on this cpu's bucket */ uma_bucket_t uc_freebucket; /* Bucket we're freeing to */ uma_bucket_t uc_allocbucket; /* Bucket to allocate from */ u_int64_t uc_allocs; /* Count of allocations */ @@ -198,8 +197,6 @@ struct uma_cache { typedef struct uma_cache * uma_cache_t; -#define LOCKNAME_LEN 16 /* Length of the name for cpu locks */ - /* * Zone management structure * @@ -207,7 +204,6 @@ typedef struct uma_cache * uma_cache_t; * */ struct uma_zone { - char uz_lname[LOCKNAME_LEN]; /* Text name for the cpu lock */ char *uz_name; /* Text name of the zone */ LIST_ENTRY(uma_zone) uz_link; /* List of all zones */ u_int32_t uz_align; /* Alignment mask */ @@ -292,26 +288,15 @@ void uma_large_free(uma_slab_t slab); #define ZONE_LOCK(z) mtx_lock(&(z)->uz_lock) #define ZONE_UNLOCK(z) mtx_unlock(&(z)->uz_lock) -#define CPU_LOCK_INIT(z, cpu, lc) \ - do { \ - if ((lc)) \ - mtx_init(&(z)->uz_cpu[(cpu)].uc_lock, \ - (z)->uz_lname, (z)->uz_lname, \ - MTX_DEF | MTX_DUPOK); \ - else \ - mtx_init(&(z)->uz_cpu[(cpu)].uc_lock, \ - (z)->uz_lname, "UMA cpu", \ - MTX_DEF | MTX_DUPOK); \ - } while (0) +#define CPU_LOCK_INIT(cpu) \ + mtx_init(&uma_pcpu_mtx[(cpu)], "UMA pcpu", "UMA pcpu", \ + MTX_DEF | MTX_DUPOK) -#define CPU_LOCK_FINI(z, cpu) \ - mtx_destroy(&(z)->uz_cpu[(cpu)].uc_lock) +#define CPU_LOCK(cpu) \ + mtx_lock(&uma_pcpu_mtx[(cpu)]) -#define CPU_LOCK(z, cpu) \ - mtx_lock(&(z)->uz_cpu[(cpu)].uc_lock) - -#define CPU_UNLOCK(z, cpu) \ - mtx_unlock(&(z)->uz_cpu[(cpu)].uc_lock) +#define CPU_UNLOCK(cpu) \ + mtx_unlock(&uma_pcpu_mtx[(cpu)]) /* * Find a slab within a hash table. This is used for OFFPAGE zones to lookup