mirror of
https://git.FreeBSD.org/src.git
synced 2024-12-05 09:14:03 +00:00
Split kerne timekeep ABI structure vdso_sv_tk out of the struct
sysentvec. This allows the timekeep data to be shared between similar ABIs which cannot share sysentvec. Make the timekeep_push_vdso() tick callback to the timekeep structures instead of sysentvecs. If several sysentvec share the vdso_sv_tk structure, we would update the userspace data several times on each tick, without the change. Only allocate vdso_sv_tk in the exec_sysvec_init() sysinit when sysentvec is marked with the new SV_TIMEKEEP flag. This saves allocation and update of unneeded vdso_sv_tk for ABIs which do not provide userspace gettimeofday yet, which are PowerPCs arches right now. Make vdso_sv_tk allocator public, namely split out and export alloc_sv_tk() and alloc_sv_tk_compat32(). ABIs which share timekeep data now can allocate it manually and share as appropriate. Requested by: nwhitehorn Tested by: nwhitehorn, pho Sponsored by: The FreeBSD Foundation MFC after: 2 weeks
This commit is contained in:
parent
c80e2a5d2b
commit
5e27d79314
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=291171
@ -75,7 +75,7 @@ struct sysentvec elf64_freebsd_sysvec = {
|
|||||||
.sv_setregs = exec_setregs,
|
.sv_setregs = exec_setregs,
|
||||||
.sv_fixlimit = NULL,
|
.sv_fixlimit = NULL,
|
||||||
.sv_maxssiz = NULL,
|
.sv_maxssiz = NULL,
|
||||||
.sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP,
|
.sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_TIMEKEEP,
|
||||||
.sv_set_syscall_retval = cpu_set_syscall_retval,
|
.sv_set_syscall_retval = cpu_set_syscall_retval,
|
||||||
.sv_fetch_syscall_args = cpu_fetch_syscall_args,
|
.sv_fetch_syscall_args = cpu_fetch_syscall_args,
|
||||||
.sv_syscallnames = syscallnames,
|
.sv_syscallnames = syscallnames,
|
||||||
|
@ -125,7 +125,7 @@ struct sysentvec ia32_freebsd_sysvec = {
|
|||||||
.sv_maxssiz = &ia32_maxssiz,
|
.sv_maxssiz = &ia32_maxssiz,
|
||||||
.sv_flags = SV_ABI_FREEBSD | SV_IA32 | SV_ILP32 |
|
.sv_flags = SV_ABI_FREEBSD | SV_IA32 | SV_ILP32 |
|
||||||
#ifdef __amd64__
|
#ifdef __amd64__
|
||||||
SV_SHP
|
SV_SHP | SV_TIMEKEEP
|
||||||
#else
|
#else
|
||||||
0
|
0
|
||||||
#endif
|
#endif
|
||||||
|
@ -81,7 +81,8 @@ struct sysentvec elf32_freebsd_sysvec = {
|
|||||||
.sv_setregs = exec_setregs,
|
.sv_setregs = exec_setregs,
|
||||||
.sv_fixlimit = NULL,
|
.sv_fixlimit = NULL,
|
||||||
.sv_maxssiz = NULL,
|
.sv_maxssiz = NULL,
|
||||||
.sv_flags = SV_ABI_FREEBSD | SV_IA32 | SV_ILP32 | SV_SHP,
|
.sv_flags = SV_ABI_FREEBSD | SV_IA32 | SV_ILP32 | SV_SHP |
|
||||||
|
SV_TIMEKEEP,
|
||||||
.sv_set_syscall_retval = cpu_set_syscall_retval,
|
.sv_set_syscall_retval = cpu_set_syscall_retval,
|
||||||
.sv_fetch_syscall_args = cpu_fetch_syscall_args,
|
.sv_fetch_syscall_args = cpu_fetch_syscall_args,
|
||||||
.sv_syscallnames = syscallnames,
|
.sv_syscallnames = syscallnames,
|
||||||
|
@ -1,7 +1,11 @@
|
|||||||
/*-
|
/*-
|
||||||
* Copyright (c) 2010, 2012 Konstantin Belousov <kib@FreeBSD.org>
|
* Copyright (c) 2010, 2012 Konstantin Belousov <kib@FreeBSD.org>
|
||||||
|
* Copyright (c) 2015 The FreeBSD Foundation
|
||||||
* All rights reserved.
|
* All rights reserved.
|
||||||
*
|
*
|
||||||
|
* Portions of this software were developed by Konstantin Belousov
|
||||||
|
* under sponsorship from the FreeBSD Foundation.
|
||||||
|
*
|
||||||
* Redistribution and use in source and binary forms, with or without
|
* Redistribution and use in source and binary forms, with or without
|
||||||
* modification, are permitted provided that the following conditions
|
* modification, are permitted provided that the following conditions
|
||||||
* are met:
|
* are met:
|
||||||
@ -34,6 +38,7 @@ __FBSDID("$FreeBSD$");
|
|||||||
#include <sys/systm.h>
|
#include <sys/systm.h>
|
||||||
#include <sys/kernel.h>
|
#include <sys/kernel.h>
|
||||||
#include <sys/lock.h>
|
#include <sys/lock.h>
|
||||||
|
#include <sys/malloc.h>
|
||||||
#include <sys/rwlock.h>
|
#include <sys/rwlock.h>
|
||||||
#include <sys/sysent.h>
|
#include <sys/sysent.h>
|
||||||
#include <sys/sysctl.h>
|
#include <sys/sysctl.h>
|
||||||
@ -127,7 +132,7 @@ SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
|
|||||||
* calls us after the timehands are updated).
|
* calls us after the timehands are updated).
|
||||||
*/
|
*/
|
||||||
static void
|
static void
|
||||||
timehands_update(struct sysentvec *sv)
|
timehands_update(struct vdso_sv_tk *svtk)
|
||||||
{
|
{
|
||||||
struct vdso_timehands th;
|
struct vdso_timehands th;
|
||||||
struct vdso_timekeep *tk;
|
struct vdso_timekeep *tk;
|
||||||
@ -135,20 +140,20 @@ timehands_update(struct sysentvec *sv)
|
|||||||
|
|
||||||
enabled = tc_fill_vdso_timehands(&th);
|
enabled = tc_fill_vdso_timehands(&th);
|
||||||
th.th_gen = 0;
|
th.th_gen = 0;
|
||||||
idx = sv->sv_timekeep_curr;
|
idx = svtk->sv_timekeep_curr;
|
||||||
if (++idx >= VDSO_TH_NUM)
|
if (++idx >= VDSO_TH_NUM)
|
||||||
idx = 0;
|
idx = 0;
|
||||||
sv->sv_timekeep_curr = idx;
|
svtk->sv_timekeep_curr = idx;
|
||||||
if (++sv->sv_timekeep_gen == 0)
|
if (++svtk->sv_timekeep_gen == 0)
|
||||||
sv->sv_timekeep_gen = 1;
|
svtk->sv_timekeep_gen = 1;
|
||||||
|
|
||||||
tk = (struct vdso_timekeep *)(shared_page_mapping +
|
tk = (struct vdso_timekeep *)(shared_page_mapping +
|
||||||
sv->sv_timekeep_off);
|
svtk->sv_timekeep_off);
|
||||||
tk->tk_th[idx].th_gen = 0;
|
tk->tk_th[idx].th_gen = 0;
|
||||||
atomic_thread_fence_rel();
|
atomic_thread_fence_rel();
|
||||||
if (enabled)
|
if (enabled)
|
||||||
tk->tk_th[idx] = th;
|
tk->tk_th[idx] = th;
|
||||||
atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
|
atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
|
||||||
atomic_store_rel_32(&tk->tk_current, idx);
|
atomic_store_rel_32(&tk->tk_current, idx);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -160,7 +165,7 @@ timehands_update(struct sysentvec *sv)
|
|||||||
|
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
static void
|
static void
|
||||||
timehands_update32(struct sysentvec *sv)
|
timehands_update32(struct vdso_sv_tk *svtk)
|
||||||
{
|
{
|
||||||
struct vdso_timehands32 th;
|
struct vdso_timehands32 th;
|
||||||
struct vdso_timekeep32 *tk;
|
struct vdso_timekeep32 *tk;
|
||||||
@ -168,20 +173,20 @@ timehands_update32(struct sysentvec *sv)
|
|||||||
|
|
||||||
enabled = tc_fill_vdso_timehands32(&th);
|
enabled = tc_fill_vdso_timehands32(&th);
|
||||||
th.th_gen = 0;
|
th.th_gen = 0;
|
||||||
idx = sv->sv_timekeep_curr;
|
idx = svtk->sv_timekeep_curr;
|
||||||
if (++idx >= VDSO_TH_NUM)
|
if (++idx >= VDSO_TH_NUM)
|
||||||
idx = 0;
|
idx = 0;
|
||||||
sv->sv_timekeep_curr = idx;
|
svtk->sv_timekeep_curr = idx;
|
||||||
if (++sv->sv_timekeep_gen == 0)
|
if (++svtk->sv_timekeep_gen == 0)
|
||||||
sv->sv_timekeep_gen = 1;
|
svtk->sv_timekeep_gen = 1;
|
||||||
|
|
||||||
tk = (struct vdso_timekeep32 *)(shared_page_mapping +
|
tk = (struct vdso_timekeep32 *)(shared_page_mapping +
|
||||||
sv->sv_timekeep_off);
|
svtk->sv_timekeep_off);
|
||||||
tk->tk_th[idx].th_gen = 0;
|
tk->tk_th[idx].th_gen = 0;
|
||||||
atomic_thread_fence_rel();
|
atomic_thread_fence_rel();
|
||||||
if (enabled)
|
if (enabled)
|
||||||
tk->tk_th[idx] = th;
|
tk->tk_th[idx] = th;
|
||||||
atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
|
atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
|
||||||
atomic_store_rel_32(&tk->tk_current, idx);
|
atomic_store_rel_32(&tk->tk_current, idx);
|
||||||
tk->tk_enabled = enabled;
|
tk->tk_enabled = enabled;
|
||||||
}
|
}
|
||||||
@ -192,33 +197,69 @@ timehands_update32(struct sysentvec *sv)
|
|||||||
* that needs to be iterated over from the hardclock interrupt
|
* that needs to be iterated over from the hardclock interrupt
|
||||||
* context.
|
* context.
|
||||||
*/
|
*/
|
||||||
static struct sysentvec *host_sysentvec;
|
static struct vdso_sv_tk *host_svtk;
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
static struct sysentvec *compat32_sysentvec;
|
static struct vdso_sv_tk *compat32_svtk;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void
|
void
|
||||||
timekeep_push_vdso(void)
|
timekeep_push_vdso(void)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (host_sysentvec != NULL && host_sysentvec->sv_timekeep_base != 0)
|
if (host_svtk != NULL)
|
||||||
timehands_update(host_sysentvec);
|
timehands_update(host_svtk);
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
if (compat32_sysentvec != NULL &&
|
if (compat32_svtk != NULL)
|
||||||
compat32_sysentvec->sv_timekeep_base != 0)
|
timehands_update32(compat32_svtk);
|
||||||
timehands_update32(compat32_sysentvec);
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct vdso_sv_tk *
|
||||||
|
alloc_sv_tk(void)
|
||||||
|
{
|
||||||
|
struct vdso_sv_tk *svtk;
|
||||||
|
int tk_base;
|
||||||
|
uint32_t tk_ver;
|
||||||
|
|
||||||
|
tk_ver = VDSO_TK_VER_CURR;
|
||||||
|
svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
|
||||||
|
tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) +
|
||||||
|
sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16);
|
||||||
|
KASSERT(tk_base != -1, ("tk_base -1 for native"));
|
||||||
|
shared_page_write(tk_base + offsetof(struct vdso_timekeep, tk_ver),
|
||||||
|
sizeof(uint32_t), &tk_ver);
|
||||||
|
svtk->sv_timekeep_off = tk_base;
|
||||||
|
timekeep_push_vdso();
|
||||||
|
return (svtk);
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef COMPAT_FREEBSD32
|
||||||
|
struct vdso_sv_tk *
|
||||||
|
alloc_sv_tk_compat32(void)
|
||||||
|
{
|
||||||
|
struct vdso_sv_tk *svtk;
|
||||||
|
int tk_base;
|
||||||
|
uint32_t tk_ver;
|
||||||
|
|
||||||
|
svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
|
||||||
|
tk_ver = VDSO_TK_VER_CURR;
|
||||||
|
tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) +
|
||||||
|
sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16);
|
||||||
|
KASSERT(tk_base != -1, ("tk_base -1 for 32bit"));
|
||||||
|
shared_page_write(tk_base + offsetof(struct vdso_timekeep32,
|
||||||
|
tk_ver), sizeof(uint32_t), &tk_ver);
|
||||||
|
svtk->sv_timekeep_off = tk_base;
|
||||||
|
timekeep_push_vdso();
|
||||||
|
return (svtk);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
void
|
void
|
||||||
exec_sysvec_init(void *param)
|
exec_sysvec_init(void *param)
|
||||||
{
|
{
|
||||||
struct sysentvec *sv;
|
struct sysentvec *sv;
|
||||||
int tk_base;
|
|
||||||
uint32_t tk_ver;
|
|
||||||
|
|
||||||
sv = (struct sysentvec *)param;
|
sv = (struct sysentvec *)param;
|
||||||
|
|
||||||
if ((sv->sv_flags & SV_SHP) == 0)
|
if ((sv->sv_flags & SV_SHP) == 0)
|
||||||
return;
|
return;
|
||||||
sv->sv_shared_page_obj = shared_page_obj;
|
sv->sv_shared_page_obj = shared_page_obj;
|
||||||
@ -226,30 +267,22 @@ exec_sysvec_init(void *param)
|
|||||||
shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode);
|
shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode);
|
||||||
if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD)
|
if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD)
|
||||||
return;
|
return;
|
||||||
tk_ver = VDSO_TK_VER_CURR;
|
if ((sv->sv_flags & SV_TIMEKEEP) != 0) {
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
if ((sv->sv_flags & SV_ILP32) != 0) {
|
if ((sv->sv_flags & SV_ILP32) != 0) {
|
||||||
tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) +
|
KASSERT(compat32_svtk == NULL,
|
||||||
sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16);
|
("Compat32 already registered"));
|
||||||
KASSERT(tk_base != -1, ("tk_base -1 for 32bit"));
|
compat32_svtk = alloc_sv_tk_compat32();
|
||||||
shared_page_write(tk_base + offsetof(struct vdso_timekeep32,
|
sv->sv_timekeep_base = sv->sv_shared_page_base +
|
||||||
tk_ver), sizeof(uint32_t), &tk_ver);
|
compat32_svtk->sv_timekeep_off;
|
||||||
KASSERT(compat32_sysentvec == 0,
|
} else {
|
||||||
("Native compat32 already registered"));
|
|
||||||
compat32_sysentvec = sv;
|
|
||||||
} else {
|
|
||||||
#endif
|
#endif
|
||||||
tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) +
|
KASSERT(host_svtk == NULL, ("Host already registered"));
|
||||||
sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16);
|
host_svtk = alloc_sv_tk();
|
||||||
KASSERT(tk_base != -1, ("tk_base -1 for native"));
|
sv->sv_timekeep_base = sv->sv_shared_page_base +
|
||||||
shared_page_write(tk_base + offsetof(struct vdso_timekeep,
|
host_svtk->sv_timekeep_off;
|
||||||
tk_ver), sizeof(uint32_t), &tk_ver);
|
|
||||||
KASSERT(host_sysentvec == 0, ("Native already registered"));
|
|
||||||
host_sysentvec = sv;
|
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
sv->sv_timekeep_base = sv->sv_shared_page_base + tk_base;
|
|
||||||
sv->sv_timekeep_off = tk_base;
|
|
||||||
timekeep_push_vdso();
|
|
||||||
}
|
}
|
||||||
|
@ -127,13 +127,10 @@ struct sysentvec {
|
|||||||
int (*sv_fetch_syscall_args)(struct thread *, struct
|
int (*sv_fetch_syscall_args)(struct thread *, struct
|
||||||
syscall_args *);
|
syscall_args *);
|
||||||
const char **sv_syscallnames;
|
const char **sv_syscallnames;
|
||||||
|
vm_offset_t sv_timekeep_base;
|
||||||
vm_offset_t sv_shared_page_base;
|
vm_offset_t sv_shared_page_base;
|
||||||
vm_offset_t sv_shared_page_len;
|
vm_offset_t sv_shared_page_len;
|
||||||
vm_offset_t sv_sigcode_base;
|
vm_offset_t sv_sigcode_base;
|
||||||
vm_offset_t sv_timekeep_base;
|
|
||||||
int sv_timekeep_off;
|
|
||||||
int sv_timekeep_curr;
|
|
||||||
uint32_t sv_timekeep_gen;
|
|
||||||
void *sv_shared_page_obj;
|
void *sv_shared_page_obj;
|
||||||
void (*sv_schedtail)(struct thread *);
|
void (*sv_schedtail)(struct thread *);
|
||||||
void (*sv_thread_detach)(struct thread *);
|
void (*sv_thread_detach)(struct thread *);
|
||||||
@ -145,6 +142,7 @@ struct sysentvec {
|
|||||||
#define SV_AOUT 0x008000 /* a.out executable. */
|
#define SV_AOUT 0x008000 /* a.out executable. */
|
||||||
#define SV_SHP 0x010000 /* Shared page. */
|
#define SV_SHP 0x010000 /* Shared page. */
|
||||||
#define SV_CAPSICUM 0x020000 /* Force cap_enter() on startup. */
|
#define SV_CAPSICUM 0x020000 /* Force cap_enter() on startup. */
|
||||||
|
#define SV_TIMEKEEP 0x040000
|
||||||
|
|
||||||
#define SV_ABI_MASK 0xff
|
#define SV_ABI_MASK 0xff
|
||||||
#define SV_PROC_FLAG(p, x) ((p)->p_sysent->sv_flags & (x))
|
#define SV_PROC_FLAG(p, x) ((p)->p_sysent->sv_flags & (x))
|
||||||
@ -274,6 +272,7 @@ int shared_page_alloc(int size, int align);
|
|||||||
int shared_page_fill(int size, int align, const void *data);
|
int shared_page_fill(int size, int align, const void *data);
|
||||||
void shared_page_write(int base, int size, const void *data);
|
void shared_page_write(int base, int size, const void *data);
|
||||||
void exec_sysvec_init(void *param);
|
void exec_sysvec_init(void *param);
|
||||||
|
void exec_inittk(void);
|
||||||
|
|
||||||
#define INIT_SYSENTVEC(name, sv) \
|
#define INIT_SYSENTVEC(name, sv) \
|
||||||
SYSINIT(name, SI_SUB_EXEC, SI_ORDER_ANY, \
|
SYSINIT(name, SI_SUB_EXEC, SI_ORDER_ANY, \
|
||||||
|
@ -71,6 +71,12 @@ int __vdso_gettimekeep(struct vdso_timekeep **tk);
|
|||||||
|
|
||||||
struct timecounter;
|
struct timecounter;
|
||||||
|
|
||||||
|
struct vdso_sv_tk {
|
||||||
|
int sv_timekeep_off;
|
||||||
|
int sv_timekeep_curr;
|
||||||
|
uint32_t sv_timekeep_gen;
|
||||||
|
};
|
||||||
|
|
||||||
void timekeep_push_vdso(void);
|
void timekeep_push_vdso(void);
|
||||||
|
|
||||||
uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
|
uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
|
||||||
@ -86,6 +92,8 @@ uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
|
|||||||
uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th,
|
uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th,
|
||||||
struct timecounter *tc);
|
struct timecounter *tc);
|
||||||
|
|
||||||
|
struct vdso_sv_tk *alloc_sv_tk(void);
|
||||||
|
|
||||||
#define VDSO_TH_NUM 4
|
#define VDSO_TH_NUM 4
|
||||||
|
|
||||||
#ifdef COMPAT_FREEBSD32
|
#ifdef COMPAT_FREEBSD32
|
||||||
@ -115,6 +123,7 @@ struct vdso_timekeep32 {
|
|||||||
uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
|
uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
|
||||||
uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32,
|
uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32,
|
||||||
struct timecounter *tc);
|
struct timecounter *tc);
|
||||||
|
struct vdso_sv_tk *alloc_sv_tk_compat32(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
#endif
|
#endif
|
||||||
|
Loading…
Reference in New Issue
Block a user