mirror of
https://git.FreeBSD.org/src.git
synced 2024-12-24 11:29:10 +00:00
92de34df2c
a pcb from stoppcbs[] rather than the thread's PCB. However, exited threads retained td_oncpu from the last time they ran, and newborn threads had their CPU fields cleared to zero during fork and thread creation since they are in the set of fields zeroed when threads are setup. To fix, explicitly update the CPU fields for exiting threads in sched_throw() to reflect the switch out and reset the CPU fields for new threads in sched_fork_thread() to NOCPU. Reviewed by: kib MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D3193
1789 lines
45 KiB
C
1789 lines
45 KiB
C
/*-
|
|
* Copyright (c) 1982, 1986, 1990, 1991, 1993
|
|
* The Regents of the University of California. All rights reserved.
|
|
* (c) UNIX System Laboratories, Inc.
|
|
* All or some portions of this file are derived from material licensed
|
|
* to the University of California by American Telephone and Telegraph
|
|
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
|
|
* the permission of UNIX System Laboratories, Inc.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
* may be used to endorse or promote products derived from this software
|
|
* without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include "opt_hwpmc_hooks.h"
|
|
#include "opt_sched.h"
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/cpuset.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/ktr.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/kthread.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/resourcevar.h>
|
|
#include <sys/sched.h>
|
|
#include <sys/sdt.h>
|
|
#include <sys/smp.h>
|
|
#include <sys/sysctl.h>
|
|
#include <sys/sx.h>
|
|
#include <sys/turnstile.h>
|
|
#include <sys/umtx.h>
|
|
#include <machine/pcb.h>
|
|
#include <machine/smp.h>
|
|
|
|
#ifdef HWPMC_HOOKS
|
|
#include <sys/pmckern.h>
|
|
#endif
|
|
|
|
#ifdef KDTRACE_HOOKS
|
|
#include <sys/dtrace_bsd.h>
|
|
int dtrace_vtime_active;
|
|
dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
|
|
#endif
|
|
|
|
/*
|
|
* INVERSE_ESTCPU_WEIGHT is only suitable for statclock() frequencies in
|
|
* the range 100-256 Hz (approximately).
|
|
*/
|
|
#define ESTCPULIM(e) \
|
|
min((e), INVERSE_ESTCPU_WEIGHT * (NICE_WEIGHT * (PRIO_MAX - PRIO_MIN) - \
|
|
RQ_PPQ) + INVERSE_ESTCPU_WEIGHT - 1)
|
|
#ifdef SMP
|
|
#define INVERSE_ESTCPU_WEIGHT (8 * smp_cpus)
|
|
#else
|
|
#define INVERSE_ESTCPU_WEIGHT 8 /* 1 / (priorities per estcpu level). */
|
|
#endif
|
|
#define NICE_WEIGHT 1 /* Priorities per nice level. */
|
|
|
|
#define TS_NAME_LEN (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))
|
|
|
|
/*
|
|
* The schedulable entity that runs a context.
|
|
* This is an extension to the thread structure and is tailored to
|
|
* the requirements of this scheduler
|
|
*/
|
|
struct td_sched {
|
|
fixpt_t ts_pctcpu; /* (j) %cpu during p_swtime. */
|
|
int ts_cpticks; /* (j) Ticks of cpu time. */
|
|
int ts_slptime; /* (j) Seconds !RUNNING. */
|
|
int ts_slice; /* Remaining part of time slice. */
|
|
int ts_flags;
|
|
struct runq *ts_runq; /* runq the thread is currently on */
|
|
#ifdef KTR
|
|
char ts_name[TS_NAME_LEN];
|
|
#endif
|
|
};
|
|
|
|
/* flags kept in td_flags */
|
|
#define TDF_DIDRUN TDF_SCHED0 /* thread actually ran. */
|
|
#define TDF_BOUND TDF_SCHED1 /* Bound to one CPU. */
|
|
#define TDF_SLICEEND TDF_SCHED2 /* Thread time slice is over. */
|
|
|
|
/* flags kept in ts_flags */
|
|
#define TSF_AFFINITY 0x0001 /* Has a non-"full" CPU set. */
|
|
|
|
#define SKE_RUNQ_PCPU(ts) \
|
|
((ts)->ts_runq != 0 && (ts)->ts_runq != &runq)
|
|
|
|
#define THREAD_CAN_SCHED(td, cpu) \
|
|
CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)
|
|
|
|
static struct td_sched td_sched0;
|
|
static struct mtx sched_lock;
|
|
|
|
static int realstathz = 127; /* stathz is sometimes 0 and run off of hz. */
|
|
static int sched_tdcnt; /* Total runnable threads in the system. */
|
|
static int sched_slice = 12; /* Thread run time before rescheduling. */
|
|
|
|
static void setup_runqs(void);
|
|
static void schedcpu(void);
|
|
static void schedcpu_thread(void);
|
|
static void sched_priority(struct thread *td, u_char prio);
|
|
static void sched_setup(void *dummy);
|
|
static void maybe_resched(struct thread *td);
|
|
static void updatepri(struct thread *td);
|
|
static void resetpriority(struct thread *td);
|
|
static void resetpriority_thread(struct thread *td);
|
|
#ifdef SMP
|
|
static int sched_pickcpu(struct thread *td);
|
|
static int forward_wakeup(int cpunum);
|
|
static void kick_other_cpu(int pri, int cpuid);
|
|
#endif
|
|
|
|
static struct kproc_desc sched_kp = {
|
|
"schedcpu",
|
|
schedcpu_thread,
|
|
NULL
|
|
};
|
|
SYSINIT(schedcpu, SI_SUB_LAST, SI_ORDER_FIRST, kproc_start,
|
|
&sched_kp);
|
|
SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);
|
|
|
|
static void sched_initticks(void *dummy);
|
|
SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
|
|
NULL);
|
|
|
|
/*
|
|
* Global run queue.
|
|
*/
|
|
static struct runq runq;
|
|
|
|
#ifdef SMP
|
|
/*
|
|
* Per-CPU run queues
|
|
*/
|
|
static struct runq runq_pcpu[MAXCPU];
|
|
long runq_length[MAXCPU];
|
|
|
|
static cpuset_t idle_cpus_mask;
|
|
#endif
|
|
|
|
struct pcpuidlestat {
|
|
u_int idlecalls;
|
|
u_int oldidlecalls;
|
|
};
|
|
static DPCPU_DEFINE(struct pcpuidlestat, idlestat);
|
|
|
|
static void
|
|
setup_runqs(void)
|
|
{
|
|
#ifdef SMP
|
|
int i;
|
|
|
|
for (i = 0; i < MAXCPU; ++i)
|
|
runq_init(&runq_pcpu[i]);
|
|
#endif
|
|
|
|
runq_init(&runq);
|
|
}
|
|
|
|
static int
|
|
sysctl_kern_quantum(SYSCTL_HANDLER_ARGS)
|
|
{
|
|
int error, new_val, period;
|
|
|
|
period = 1000000 / realstathz;
|
|
new_val = period * sched_slice;
|
|
error = sysctl_handle_int(oidp, &new_val, 0, req);
|
|
if (error != 0 || req->newptr == NULL)
|
|
return (error);
|
|
if (new_val <= 0)
|
|
return (EINVAL);
|
|
sched_slice = imax(1, (new_val + period / 2) / period);
|
|
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
|
|
realstathz);
|
|
return (0);
|
|
}
|
|
|
|
SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD, 0, "Scheduler");
|
|
|
|
SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, "4BSD", 0,
|
|
"Scheduler name");
|
|
SYSCTL_PROC(_kern_sched, OID_AUTO, quantum, CTLTYPE_INT | CTLFLAG_RW,
|
|
NULL, 0, sysctl_kern_quantum, "I",
|
|
"Quantum for timeshare threads in microseconds");
|
|
SYSCTL_INT(_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0,
|
|
"Quantum for timeshare threads in stathz ticks");
|
|
#ifdef SMP
|
|
/* Enable forwarding of wakeups to all other cpus */
|
|
static SYSCTL_NODE(_kern_sched, OID_AUTO, ipiwakeup, CTLFLAG_RD, NULL,
|
|
"Kernel SMP");
|
|
|
|
static int runq_fuzz = 1;
|
|
SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, "");
|
|
|
|
static int forward_wakeup_enabled = 1;
|
|
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, enabled, CTLFLAG_RW,
|
|
&forward_wakeup_enabled, 0,
|
|
"Forwarding of wakeup to idle CPUs");
|
|
|
|
static int forward_wakeups_requested = 0;
|
|
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, requested, CTLFLAG_RD,
|
|
&forward_wakeups_requested, 0,
|
|
"Requests for Forwarding of wakeup to idle CPUs");
|
|
|
|
static int forward_wakeups_delivered = 0;
|
|
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, delivered, CTLFLAG_RD,
|
|
&forward_wakeups_delivered, 0,
|
|
"Completed Forwarding of wakeup to idle CPUs");
|
|
|
|
static int forward_wakeup_use_mask = 1;
|
|
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, usemask, CTLFLAG_RW,
|
|
&forward_wakeup_use_mask, 0,
|
|
"Use the mask of idle cpus");
|
|
|
|
static int forward_wakeup_use_loop = 0;
|
|
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, useloop, CTLFLAG_RW,
|
|
&forward_wakeup_use_loop, 0,
|
|
"Use a loop to find idle cpus");
|
|
|
|
#endif
|
|
#if 0
|
|
static int sched_followon = 0;
|
|
SYSCTL_INT(_kern_sched, OID_AUTO, followon, CTLFLAG_RW,
|
|
&sched_followon, 0,
|
|
"allow threads to share a quantum");
|
|
#endif
|
|
|
|
SDT_PROVIDER_DEFINE(sched);
|
|
|
|
SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *",
|
|
"struct proc *", "uint8_t");
|
|
SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *",
|
|
"struct proc *", "void *");
|
|
SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *",
|
|
"struct proc *", "void *", "int");
|
|
SDT_PROBE_DEFINE4(sched, , , lend__pri, "struct thread *",
|
|
"struct proc *", "uint8_t", "struct thread *");
|
|
SDT_PROBE_DEFINE2(sched, , , load__change, "int", "int");
|
|
SDT_PROBE_DEFINE2(sched, , , off__cpu, "struct thread *",
|
|
"struct proc *");
|
|
SDT_PROBE_DEFINE(sched, , , on__cpu);
|
|
SDT_PROBE_DEFINE(sched, , , remain__cpu);
|
|
SDT_PROBE_DEFINE2(sched, , , surrender, "struct thread *",
|
|
"struct proc *");
|
|
|
|
static __inline void
|
|
sched_load_add(void)
|
|
{
|
|
|
|
sched_tdcnt++;
|
|
KTR_COUNTER0(KTR_SCHED, "load", "global load", sched_tdcnt);
|
|
SDT_PROBE2(sched, , , load__change, NOCPU, sched_tdcnt);
|
|
}
|
|
|
|
static __inline void
|
|
sched_load_rem(void)
|
|
{
|
|
|
|
sched_tdcnt--;
|
|
KTR_COUNTER0(KTR_SCHED, "load", "global load", sched_tdcnt);
|
|
SDT_PROBE2(sched, , , load__change, NOCPU, sched_tdcnt);
|
|
}
|
|
/*
|
|
* Arrange to reschedule if necessary, taking the priorities and
|
|
* schedulers into account.
|
|
*/
|
|
static void
|
|
maybe_resched(struct thread *td)
|
|
{
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
if (td->td_priority < curthread->td_priority)
|
|
curthread->td_flags |= TDF_NEEDRESCHED;
|
|
}
|
|
|
|
/*
|
|
* This function is called when a thread is about to be put on run queue
|
|
* because it has been made runnable or its priority has been adjusted. It
|
|
* determines if the new thread should be immediately preempted to. If so,
|
|
* it switches to it and eventually returns true. If not, it returns false
|
|
* so that the caller may place the thread on an appropriate run queue.
|
|
*/
|
|
int
|
|
maybe_preempt(struct thread *td)
|
|
{
|
|
#ifdef PREEMPTION
|
|
struct thread *ctd;
|
|
int cpri, pri;
|
|
|
|
/*
|
|
* The new thread should not preempt the current thread if any of the
|
|
* following conditions are true:
|
|
*
|
|
* - The kernel is in the throes of crashing (panicstr).
|
|
* - The current thread has a higher (numerically lower) or
|
|
* equivalent priority. Note that this prevents curthread from
|
|
* trying to preempt to itself.
|
|
* - It is too early in the boot for context switches (cold is set).
|
|
* - The current thread has an inhibitor set or is in the process of
|
|
* exiting. In this case, the current thread is about to switch
|
|
* out anyways, so there's no point in preempting. If we did,
|
|
* the current thread would not be properly resumed as well, so
|
|
* just avoid that whole landmine.
|
|
* - If the new thread's priority is not a realtime priority and
|
|
* the current thread's priority is not an idle priority and
|
|
* FULL_PREEMPTION is disabled.
|
|
*
|
|
* If all of these conditions are false, but the current thread is in
|
|
* a nested critical section, then we have to defer the preemption
|
|
* until we exit the critical section. Otherwise, switch immediately
|
|
* to the new thread.
|
|
*/
|
|
ctd = curthread;
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
KASSERT((td->td_inhibitors == 0),
|
|
("maybe_preempt: trying to run inhibited thread"));
|
|
pri = td->td_priority;
|
|
cpri = ctd->td_priority;
|
|
if (panicstr != NULL || pri >= cpri || cold /* || dumping */ ||
|
|
TD_IS_INHIBITED(ctd))
|
|
return (0);
|
|
#ifndef FULL_PREEMPTION
|
|
if (pri > PRI_MAX_ITHD && cpri < PRI_MIN_IDLE)
|
|
return (0);
|
|
#endif
|
|
|
|
if (ctd->td_critnest > 1) {
|
|
CTR1(KTR_PROC, "maybe_preempt: in critical section %d",
|
|
ctd->td_critnest);
|
|
ctd->td_owepreempt = 1;
|
|
return (0);
|
|
}
|
|
/*
|
|
* Thread is runnable but not yet put on system run queue.
|
|
*/
|
|
MPASS(ctd->td_lock == td->td_lock);
|
|
MPASS(TD_ON_RUNQ(td));
|
|
TD_SET_RUNNING(td);
|
|
CTR3(KTR_PROC, "preempting to thread %p (pid %d, %s)\n", td,
|
|
td->td_proc->p_pid, td->td_name);
|
|
mi_switch(SW_INVOL | SW_PREEMPT | SWT_PREEMPT, td);
|
|
/*
|
|
* td's lock pointer may have changed. We have to return with it
|
|
* locked.
|
|
*/
|
|
spinlock_enter();
|
|
thread_unlock(ctd);
|
|
thread_lock(td);
|
|
spinlock_exit();
|
|
return (1);
|
|
#else
|
|
return (0);
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Constants for digital decay and forget:
|
|
* 90% of (td_estcpu) usage in 5 * loadav time
|
|
* 95% of (ts_pctcpu) usage in 60 seconds (load insensitive)
|
|
* Note that, as ps(1) mentions, this can let percentages
|
|
* total over 100% (I've seen 137.9% for 3 processes).
|
|
*
|
|
* Note that schedclock() updates td_estcpu and p_cpticks asynchronously.
|
|
*
|
|
* We wish to decay away 90% of td_estcpu in (5 * loadavg) seconds.
|
|
* That is, the system wants to compute a value of decay such
|
|
* that the following for loop:
|
|
* for (i = 0; i < (5 * loadavg); i++)
|
|
* td_estcpu *= decay;
|
|
* will compute
|
|
* td_estcpu *= 0.1;
|
|
* for all values of loadavg:
|
|
*
|
|
* Mathematically this loop can be expressed by saying:
|
|
* decay ** (5 * loadavg) ~= .1
|
|
*
|
|
* The system computes decay as:
|
|
* decay = (2 * loadavg) / (2 * loadavg + 1)
|
|
*
|
|
* We wish to prove that the system's computation of decay
|
|
* will always fulfill the equation:
|
|
* decay ** (5 * loadavg) ~= .1
|
|
*
|
|
* If we compute b as:
|
|
* b = 2 * loadavg
|
|
* then
|
|
* decay = b / (b + 1)
|
|
*
|
|
* We now need to prove two things:
|
|
* 1) Given factor ** (5 * loadavg) ~= .1, prove factor == b/(b+1)
|
|
* 2) Given b/(b+1) ** power ~= .1, prove power == (5 * loadavg)
|
|
*
|
|
* Facts:
|
|
* For x close to zero, exp(x) =~ 1 + x, since
|
|
* exp(x) = 0! + x**1/1! + x**2/2! + ... .
|
|
* therefore exp(-1/b) =~ 1 - (1/b) = (b-1)/b.
|
|
* For x close to zero, ln(1+x) =~ x, since
|
|
* ln(1+x) = x - x**2/2 + x**3/3 - ... -1 < x < 1
|
|
* therefore ln(b/(b+1)) = ln(1 - 1/(b+1)) =~ -1/(b+1).
|
|
* ln(.1) =~ -2.30
|
|
*
|
|
* Proof of (1):
|
|
* Solve (factor)**(power) =~ .1 given power (5*loadav):
|
|
* solving for factor,
|
|
* ln(factor) =~ (-2.30/5*loadav), or
|
|
* factor =~ exp(-1/((5/2.30)*loadav)) =~ exp(-1/(2*loadav)) =
|
|
* exp(-1/b) =~ (b-1)/b =~ b/(b+1). QED
|
|
*
|
|
* Proof of (2):
|
|
* Solve (factor)**(power) =~ .1 given factor == (b/(b+1)):
|
|
* solving for power,
|
|
* power*ln(b/(b+1)) =~ -2.30, or
|
|
* power =~ 2.3 * (b + 1) = 4.6*loadav + 2.3 =~ 5*loadav. QED
|
|
*
|
|
* Actual power values for the implemented algorithm are as follows:
|
|
* loadav: 1 2 3 4
|
|
* power: 5.68 10.32 14.94 19.55
|
|
*/
|
|
|
|
/* calculations for digital decay to forget 90% of usage in 5*loadav sec */
|
|
#define loadfactor(loadav) (2 * (loadav))
|
|
#define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE))
|
|
|
|
/* decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */
|
|
static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */
|
|
SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, "");
|
|
|
|
/*
|
|
* If `ccpu' is not equal to `exp(-1/20)' and you still want to use the
|
|
* faster/more-accurate formula, you'll have to estimate CCPU_SHIFT below
|
|
* and possibly adjust FSHIFT in "param.h" so that (FSHIFT >= CCPU_SHIFT).
|
|
*
|
|
* To estimate CCPU_SHIFT for exp(-1/20), the following formula was used:
|
|
* 1 - exp(-1/20) ~= 0.0487 ~= 0.0488 == 1 (fixed pt, *11* bits).
|
|
*
|
|
* If you don't want to bother with the faster/more-accurate formula, you
|
|
* can set CCPU_SHIFT to (FSHIFT + 1) which will use a slower/less-accurate
|
|
* (more general) method of calculating the %age of CPU used by a process.
|
|
*/
|
|
#define CCPU_SHIFT 11
|
|
|
|
/*
|
|
* Recompute process priorities, every hz ticks.
|
|
* MP-safe, called without the Giant mutex.
|
|
*/
|
|
/* ARGSUSED */
|
|
static void
|
|
schedcpu(void)
|
|
{
|
|
register fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
|
|
struct thread *td;
|
|
struct proc *p;
|
|
struct td_sched *ts;
|
|
int awake;
|
|
|
|
sx_slock(&allproc_lock);
|
|
FOREACH_PROC_IN_SYSTEM(p) {
|
|
PROC_LOCK(p);
|
|
if (p->p_state == PRS_NEW) {
|
|
PROC_UNLOCK(p);
|
|
continue;
|
|
}
|
|
FOREACH_THREAD_IN_PROC(p, td) {
|
|
awake = 0;
|
|
thread_lock(td);
|
|
ts = td->td_sched;
|
|
/*
|
|
* Increment sleep time (if sleeping). We
|
|
* ignore overflow, as above.
|
|
*/
|
|
/*
|
|
* The td_sched slptimes are not touched in wakeup
|
|
* because the thread may not HAVE everything in
|
|
* memory? XXX I think this is out of date.
|
|
*/
|
|
if (TD_ON_RUNQ(td)) {
|
|
awake = 1;
|
|
td->td_flags &= ~TDF_DIDRUN;
|
|
} else if (TD_IS_RUNNING(td)) {
|
|
awake = 1;
|
|
/* Do not clear TDF_DIDRUN */
|
|
} else if (td->td_flags & TDF_DIDRUN) {
|
|
awake = 1;
|
|
td->td_flags &= ~TDF_DIDRUN;
|
|
}
|
|
|
|
/*
|
|
* ts_pctcpu is only for ps and ttyinfo().
|
|
*/
|
|
ts->ts_pctcpu = (ts->ts_pctcpu * ccpu) >> FSHIFT;
|
|
/*
|
|
* If the td_sched has been idle the entire second,
|
|
* stop recalculating its priority until
|
|
* it wakes up.
|
|
*/
|
|
if (ts->ts_cpticks != 0) {
|
|
#if (FSHIFT >= CCPU_SHIFT)
|
|
ts->ts_pctcpu += (realstathz == 100)
|
|
? ((fixpt_t) ts->ts_cpticks) <<
|
|
(FSHIFT - CCPU_SHIFT) :
|
|
100 * (((fixpt_t) ts->ts_cpticks)
|
|
<< (FSHIFT - CCPU_SHIFT)) / realstathz;
|
|
#else
|
|
ts->ts_pctcpu += ((FSCALE - ccpu) *
|
|
(ts->ts_cpticks *
|
|
FSCALE / realstathz)) >> FSHIFT;
|
|
#endif
|
|
ts->ts_cpticks = 0;
|
|
}
|
|
/*
|
|
* If there are ANY running threads in this process,
|
|
* then don't count it as sleeping.
|
|
* XXX: this is broken.
|
|
*/
|
|
if (awake) {
|
|
if (ts->ts_slptime > 1) {
|
|
/*
|
|
* In an ideal world, this should not
|
|
* happen, because whoever woke us
|
|
* up from the long sleep should have
|
|
* unwound the slptime and reset our
|
|
* priority before we run at the stale
|
|
* priority. Should KASSERT at some
|
|
* point when all the cases are fixed.
|
|
*/
|
|
updatepri(td);
|
|
}
|
|
ts->ts_slptime = 0;
|
|
} else
|
|
ts->ts_slptime++;
|
|
if (ts->ts_slptime > 1) {
|
|
thread_unlock(td);
|
|
continue;
|
|
}
|
|
td->td_estcpu = decay_cpu(loadfac, td->td_estcpu);
|
|
resetpriority(td);
|
|
resetpriority_thread(td);
|
|
thread_unlock(td);
|
|
}
|
|
PROC_UNLOCK(p);
|
|
}
|
|
sx_sunlock(&allproc_lock);
|
|
}
|
|
|
|
/*
|
|
* Main loop for a kthread that executes schedcpu once a second.
|
|
*/
|
|
static void
|
|
schedcpu_thread(void)
|
|
{
|
|
|
|
for (;;) {
|
|
schedcpu();
|
|
pause("-", hz);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Recalculate the priority of a process after it has slept for a while.
|
|
* For all load averages >= 1 and max td_estcpu of 255, sleeping for at
|
|
* least six times the loadfactor will decay td_estcpu to zero.
|
|
*/
|
|
static void
|
|
updatepri(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
fixpt_t loadfac;
|
|
unsigned int newcpu;
|
|
|
|
ts = td->td_sched;
|
|
loadfac = loadfactor(averunnable.ldavg[0]);
|
|
if (ts->ts_slptime > 5 * loadfac)
|
|
td->td_estcpu = 0;
|
|
else {
|
|
newcpu = td->td_estcpu;
|
|
ts->ts_slptime--; /* was incremented in schedcpu() */
|
|
while (newcpu && --ts->ts_slptime)
|
|
newcpu = decay_cpu(loadfac, newcpu);
|
|
td->td_estcpu = newcpu;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Compute the priority of a process when running in user mode.
|
|
* Arrange to reschedule if the resulting priority is better
|
|
* than that of the current process.
|
|
*/
|
|
static void
|
|
resetpriority(struct thread *td)
|
|
{
|
|
register unsigned int newpriority;
|
|
|
|
if (td->td_pri_class == PRI_TIMESHARE) {
|
|
newpriority = PUSER + td->td_estcpu / INVERSE_ESTCPU_WEIGHT +
|
|
NICE_WEIGHT * (td->td_proc->p_nice - PRIO_MIN);
|
|
newpriority = min(max(newpriority, PRI_MIN_TIMESHARE),
|
|
PRI_MAX_TIMESHARE);
|
|
sched_user_prio(td, newpriority);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Update the thread's priority when the associated process's user
|
|
* priority changes.
|
|
*/
|
|
static void
|
|
resetpriority_thread(struct thread *td)
|
|
{
|
|
|
|
/* Only change threads with a time sharing user priority. */
|
|
if (td->td_priority < PRI_MIN_TIMESHARE ||
|
|
td->td_priority > PRI_MAX_TIMESHARE)
|
|
return;
|
|
|
|
/* XXX the whole needresched thing is broken, but not silly. */
|
|
maybe_resched(td);
|
|
|
|
sched_prio(td, td->td_user_pri);
|
|
}
|
|
|
|
/* ARGSUSED */
|
|
static void
|
|
sched_setup(void *dummy)
|
|
{
|
|
|
|
setup_runqs();
|
|
|
|
/* Account for thread0. */
|
|
sched_load_add();
|
|
}
|
|
|
|
/*
|
|
* This routine determines time constants after stathz and hz are setup.
|
|
*/
|
|
static void
|
|
sched_initticks(void *dummy)
|
|
{
|
|
|
|
realstathz = stathz ? stathz : hz;
|
|
sched_slice = realstathz / 10; /* ~100ms */
|
|
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
|
|
realstathz);
|
|
}
|
|
|
|
/* External interfaces start here */
|
|
|
|
/*
|
|
* Very early in the boot some setup of scheduler-specific
|
|
* parts of proc0 and of some scheduler resources needs to be done.
|
|
* Called from:
|
|
* proc0_init()
|
|
*/
|
|
void
|
|
schedinit(void)
|
|
{
|
|
/*
|
|
* Set up the scheduler specific parts of proc0.
|
|
*/
|
|
proc0.p_sched = NULL; /* XXX */
|
|
thread0.td_sched = &td_sched0;
|
|
thread0.td_lock = &sched_lock;
|
|
td_sched0.ts_slice = sched_slice;
|
|
mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE);
|
|
}
|
|
|
|
int
|
|
sched_runnable(void)
|
|
{
|
|
#ifdef SMP
|
|
return runq_check(&runq) + runq_check(&runq_pcpu[PCPU_GET(cpuid)]);
|
|
#else
|
|
return runq_check(&runq);
|
|
#endif
|
|
}
|
|
|
|
int
|
|
sched_rr_interval(void)
|
|
{
|
|
|
|
/* Convert sched_slice from stathz to hz. */
|
|
return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz));
|
|
}
|
|
|
|
/*
|
|
* We adjust the priority of the current process. The priority of
|
|
* a process gets worse as it accumulates CPU time. The cpu usage
|
|
* estimator (td_estcpu) is increased here. resetpriority() will
|
|
* compute a different priority each time td_estcpu increases by
|
|
* INVERSE_ESTCPU_WEIGHT
|
|
* (until MAXPRI is reached). The cpu usage estimator ramps up
|
|
* quite quickly when the process is running (linearly), and decays
|
|
* away exponentially, at a rate which is proportionally slower when
|
|
* the system is busy. The basic principle is that the system will
|
|
* 90% forget that the process used a lot of CPU time in 5 * loadav
|
|
* seconds. This causes the system to favor processes which haven't
|
|
* run much recently, and to round-robin among other processes.
|
|
*/
|
|
void
|
|
sched_clock(struct thread *td)
|
|
{
|
|
struct pcpuidlestat *stat;
|
|
struct td_sched *ts;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
ts = td->td_sched;
|
|
|
|
ts->ts_cpticks++;
|
|
td->td_estcpu = ESTCPULIM(td->td_estcpu + 1);
|
|
if ((td->td_estcpu % INVERSE_ESTCPU_WEIGHT) == 0) {
|
|
resetpriority(td);
|
|
resetpriority_thread(td);
|
|
}
|
|
|
|
/*
|
|
* Force a context switch if the current thread has used up a full
|
|
* time slice (default is 100ms).
|
|
*/
|
|
if (!TD_IS_IDLETHREAD(td) && --ts->ts_slice <= 0) {
|
|
ts->ts_slice = sched_slice;
|
|
td->td_flags |= TDF_NEEDRESCHED | TDF_SLICEEND;
|
|
}
|
|
|
|
stat = DPCPU_PTR(idlestat);
|
|
stat->oldidlecalls = stat->idlecalls;
|
|
stat->idlecalls = 0;
|
|
}
|
|
|
|
/*
|
|
* Charge child's scheduling CPU usage to parent.
|
|
*/
|
|
void
|
|
sched_exit(struct proc *p, struct thread *td)
|
|
{
|
|
|
|
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "proc exit",
|
|
"prio:%d", td->td_priority);
|
|
|
|
PROC_LOCK_ASSERT(p, MA_OWNED);
|
|
sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
|
|
}
|
|
|
|
void
|
|
sched_exit_thread(struct thread *td, struct thread *child)
|
|
{
|
|
|
|
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "exit",
|
|
"prio:%d", child->td_priority);
|
|
thread_lock(td);
|
|
td->td_estcpu = ESTCPULIM(td->td_estcpu + child->td_estcpu);
|
|
thread_unlock(td);
|
|
thread_lock(child);
|
|
if ((child->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_rem();
|
|
thread_unlock(child);
|
|
}
|
|
|
|
void
|
|
sched_fork(struct thread *td, struct thread *childtd)
|
|
{
|
|
sched_fork_thread(td, childtd);
|
|
}
|
|
|
|
void
|
|
sched_fork_thread(struct thread *td, struct thread *childtd)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
childtd->td_oncpu = NOCPU;
|
|
childtd->td_lastcpu = NOCPU;
|
|
childtd->td_estcpu = td->td_estcpu;
|
|
childtd->td_lock = &sched_lock;
|
|
childtd->td_cpuset = cpuset_ref(td->td_cpuset);
|
|
childtd->td_priority = childtd->td_base_pri;
|
|
ts = childtd->td_sched;
|
|
bzero(ts, sizeof(*ts));
|
|
ts->ts_flags |= (td->td_sched->ts_flags & TSF_AFFINITY);
|
|
ts->ts_slice = 1;
|
|
}
|
|
|
|
void
|
|
sched_nice(struct proc *p, int nice)
|
|
{
|
|
struct thread *td;
|
|
|
|
PROC_LOCK_ASSERT(p, MA_OWNED);
|
|
p->p_nice = nice;
|
|
FOREACH_THREAD_IN_PROC(p, td) {
|
|
thread_lock(td);
|
|
resetpriority(td);
|
|
resetpriority_thread(td);
|
|
thread_unlock(td);
|
|
}
|
|
}
|
|
|
|
void
|
|
sched_class(struct thread *td, int class)
|
|
{
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
td->td_pri_class = class;
|
|
}
|
|
|
|
/*
|
|
* Adjust the priority of a thread.
|
|
*/
|
|
static void
|
|
sched_priority(struct thread *td, u_char prio)
|
|
{
|
|
|
|
|
|
KTR_POINT3(KTR_SCHED, "thread", sched_tdname(td), "priority change",
|
|
"prio:%d", td->td_priority, "new prio:%d", prio, KTR_ATTR_LINKED,
|
|
sched_tdname(curthread));
|
|
SDT_PROBE3(sched, , , change__pri, td, td->td_proc, prio);
|
|
if (td != curthread && prio > td->td_priority) {
|
|
KTR_POINT3(KTR_SCHED, "thread", sched_tdname(curthread),
|
|
"lend prio", "prio:%d", td->td_priority, "new prio:%d",
|
|
prio, KTR_ATTR_LINKED, sched_tdname(td));
|
|
SDT_PROBE4(sched, , , lend__pri, td, td->td_proc, prio,
|
|
curthread);
|
|
}
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
if (td->td_priority == prio)
|
|
return;
|
|
td->td_priority = prio;
|
|
if (TD_ON_RUNQ(td) && td->td_rqindex != (prio / RQ_PPQ)) {
|
|
sched_rem(td);
|
|
sched_add(td, SRQ_BORING);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Update a thread's priority when it is lent another thread's
|
|
* priority.
|
|
*/
|
|
void
|
|
sched_lend_prio(struct thread *td, u_char prio)
|
|
{
|
|
|
|
td->td_flags |= TDF_BORROWING;
|
|
sched_priority(td, prio);
|
|
}
|
|
|
|
/*
|
|
* Restore a thread's priority when priority propagation is
|
|
* over. The prio argument is the minimum priority the thread
|
|
* needs to have to satisfy other possible priority lending
|
|
* requests. If the thread's regulary priority is less
|
|
* important than prio the thread will keep a priority boost
|
|
* of prio.
|
|
*/
|
|
void
|
|
sched_unlend_prio(struct thread *td, u_char prio)
|
|
{
|
|
u_char base_pri;
|
|
|
|
if (td->td_base_pri >= PRI_MIN_TIMESHARE &&
|
|
td->td_base_pri <= PRI_MAX_TIMESHARE)
|
|
base_pri = td->td_user_pri;
|
|
else
|
|
base_pri = td->td_base_pri;
|
|
if (prio >= base_pri) {
|
|
td->td_flags &= ~TDF_BORROWING;
|
|
sched_prio(td, base_pri);
|
|
} else
|
|
sched_lend_prio(td, prio);
|
|
}
|
|
|
|
void
|
|
sched_prio(struct thread *td, u_char prio)
|
|
{
|
|
u_char oldprio;
|
|
|
|
/* First, update the base priority. */
|
|
td->td_base_pri = prio;
|
|
|
|
/*
|
|
* If the thread is borrowing another thread's priority, don't ever
|
|
* lower the priority.
|
|
*/
|
|
if (td->td_flags & TDF_BORROWING && td->td_priority < prio)
|
|
return;
|
|
|
|
/* Change the real priority. */
|
|
oldprio = td->td_priority;
|
|
sched_priority(td, prio);
|
|
|
|
/*
|
|
* If the thread is on a turnstile, then let the turnstile update
|
|
* its state.
|
|
*/
|
|
if (TD_ON_LOCK(td) && oldprio != prio)
|
|
turnstile_adjust(td, oldprio);
|
|
}
|
|
|
|
void
|
|
sched_user_prio(struct thread *td, u_char prio)
|
|
{
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
td->td_base_user_pri = prio;
|
|
if (td->td_lend_user_pri <= prio)
|
|
return;
|
|
td->td_user_pri = prio;
|
|
}
|
|
|
|
void
|
|
sched_lend_user_prio(struct thread *td, u_char prio)
|
|
{
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
td->td_lend_user_pri = prio;
|
|
td->td_user_pri = min(prio, td->td_base_user_pri);
|
|
if (td->td_priority > td->td_user_pri)
|
|
sched_prio(td, td->td_user_pri);
|
|
else if (td->td_priority != td->td_user_pri)
|
|
td->td_flags |= TDF_NEEDRESCHED;
|
|
}
|
|
|
|
void
|
|
sched_sleep(struct thread *td, int pri)
|
|
{
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
td->td_slptick = ticks;
|
|
td->td_sched->ts_slptime = 0;
|
|
if (pri != 0 && PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
|
|
sched_prio(td, pri);
|
|
if (TD_IS_SUSPENDED(td) || pri >= PSOCK)
|
|
td->td_flags |= TDF_CANSWAP;
|
|
}
|
|
|
|
void
|
|
sched_switch(struct thread *td, struct thread *newtd, int flags)
|
|
{
|
|
struct mtx *tmtx;
|
|
struct td_sched *ts;
|
|
struct proc *p;
|
|
int preempted;
|
|
|
|
tmtx = NULL;
|
|
ts = td->td_sched;
|
|
p = td->td_proc;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
|
|
/*
|
|
* Switch to the sched lock to fix things up and pick
|
|
* a new thread.
|
|
* Block the td_lock in order to avoid breaking the critical path.
|
|
*/
|
|
if (td->td_lock != &sched_lock) {
|
|
mtx_lock_spin(&sched_lock);
|
|
tmtx = thread_lock_block(td);
|
|
}
|
|
|
|
if ((td->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_rem();
|
|
|
|
td->td_lastcpu = td->td_oncpu;
|
|
preempted = !((td->td_flags & TDF_SLICEEND) ||
|
|
(flags & SWT_RELINQUISH));
|
|
td->td_flags &= ~(TDF_NEEDRESCHED | TDF_SLICEEND);
|
|
td->td_owepreempt = 0;
|
|
td->td_oncpu = NOCPU;
|
|
|
|
/*
|
|
* At the last moment, if this thread is still marked RUNNING,
|
|
* then put it back on the run queue as it has not been suspended
|
|
* or stopped or any thing else similar. We never put the idle
|
|
* threads on the run queue, however.
|
|
*/
|
|
if (td->td_flags & TDF_IDLETD) {
|
|
TD_SET_CAN_RUN(td);
|
|
#ifdef SMP
|
|
CPU_CLR(PCPU_GET(cpuid), &idle_cpus_mask);
|
|
#endif
|
|
} else {
|
|
if (TD_IS_RUNNING(td)) {
|
|
/* Put us back on the run queue. */
|
|
sched_add(td, preempted ?
|
|
SRQ_OURSELF|SRQ_YIELDING|SRQ_PREEMPTED :
|
|
SRQ_OURSELF|SRQ_YIELDING);
|
|
}
|
|
}
|
|
if (newtd) {
|
|
/*
|
|
* The thread we are about to run needs to be counted
|
|
* as if it had been added to the run queue and selected.
|
|
* It came from:
|
|
* * A preemption
|
|
* * An upcall
|
|
* * A followon
|
|
*/
|
|
KASSERT((newtd->td_inhibitors == 0),
|
|
("trying to run inhibited thread"));
|
|
newtd->td_flags |= TDF_DIDRUN;
|
|
TD_SET_RUNNING(newtd);
|
|
if ((newtd->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_add();
|
|
} else {
|
|
newtd = choosethread();
|
|
MPASS(newtd->td_lock == &sched_lock);
|
|
}
|
|
|
|
if (td != newtd) {
|
|
#ifdef HWPMC_HOOKS
|
|
if (PMC_PROC_IS_USING_PMCS(td->td_proc))
|
|
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
|
|
#endif
|
|
|
|
SDT_PROBE2(sched, , , off__cpu, newtd, newtd->td_proc);
|
|
|
|
/* I feel sleepy */
|
|
lock_profile_release_lock(&sched_lock.lock_object);
|
|
#ifdef KDTRACE_HOOKS
|
|
/*
|
|
* If DTrace has set the active vtime enum to anything
|
|
* other than INACTIVE (0), then it should have set the
|
|
* function to call.
|
|
*/
|
|
if (dtrace_vtime_active)
|
|
(*dtrace_vtime_switch_func)(newtd);
|
|
#endif
|
|
|
|
cpu_switch(td, newtd, tmtx != NULL ? tmtx : td->td_lock);
|
|
lock_profile_obtain_lock_success(&sched_lock.lock_object,
|
|
0, 0, __FILE__, __LINE__);
|
|
/*
|
|
* Where am I? What year is it?
|
|
* We are in the same thread that went to sleep above,
|
|
* but any amount of time may have passed. All our context
|
|
* will still be available as will local variables.
|
|
* PCPU values however may have changed as we may have
|
|
* changed CPU so don't trust cached values of them.
|
|
* New threads will go to fork_exit() instead of here
|
|
* so if you change things here you may need to change
|
|
* things there too.
|
|
*
|
|
* If the thread above was exiting it will never wake
|
|
* up again here, so either it has saved everything it
|
|
* needed to, or the thread_wait() or wait() will
|
|
* need to reap it.
|
|
*/
|
|
|
|
SDT_PROBE0(sched, , , on__cpu);
|
|
#ifdef HWPMC_HOOKS
|
|
if (PMC_PROC_IS_USING_PMCS(td->td_proc))
|
|
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
|
|
#endif
|
|
} else
|
|
SDT_PROBE0(sched, , , remain__cpu);
|
|
|
|
#ifdef SMP
|
|
if (td->td_flags & TDF_IDLETD)
|
|
CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask);
|
|
#endif
|
|
sched_lock.mtx_lock = (uintptr_t)td;
|
|
td->td_oncpu = PCPU_GET(cpuid);
|
|
MPASS(td->td_lock == &sched_lock);
|
|
}
|
|
|
|
void
|
|
sched_wakeup(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
ts = td->td_sched;
|
|
td->td_flags &= ~TDF_CANSWAP;
|
|
if (ts->ts_slptime > 1) {
|
|
updatepri(td);
|
|
resetpriority(td);
|
|
}
|
|
td->td_slptick = 0;
|
|
ts->ts_slptime = 0;
|
|
ts->ts_slice = sched_slice;
|
|
sched_add(td, SRQ_BORING);
|
|
}
|
|
|
|
#ifdef SMP
|
|
static int
|
|
forward_wakeup(int cpunum)
|
|
{
|
|
struct pcpu *pc;
|
|
cpuset_t dontuse, map, map2;
|
|
u_int id, me;
|
|
int iscpuset;
|
|
|
|
mtx_assert(&sched_lock, MA_OWNED);
|
|
|
|
CTR0(KTR_RUNQ, "forward_wakeup()");
|
|
|
|
if ((!forward_wakeup_enabled) ||
|
|
(forward_wakeup_use_mask == 0 && forward_wakeup_use_loop == 0))
|
|
return (0);
|
|
if (!smp_started || cold || panicstr)
|
|
return (0);
|
|
|
|
forward_wakeups_requested++;
|
|
|
|
/*
|
|
* Check the idle mask we received against what we calculated
|
|
* before in the old version.
|
|
*/
|
|
me = PCPU_GET(cpuid);
|
|
|
|
/* Don't bother if we should be doing it ourself. */
|
|
if (CPU_ISSET(me, &idle_cpus_mask) &&
|
|
(cpunum == NOCPU || me == cpunum))
|
|
return (0);
|
|
|
|
CPU_SETOF(me, &dontuse);
|
|
CPU_OR(&dontuse, &stopped_cpus);
|
|
CPU_OR(&dontuse, &hlt_cpus_mask);
|
|
CPU_ZERO(&map2);
|
|
if (forward_wakeup_use_loop) {
|
|
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
|
|
id = pc->pc_cpuid;
|
|
if (!CPU_ISSET(id, &dontuse) &&
|
|
pc->pc_curthread == pc->pc_idlethread) {
|
|
CPU_SET(id, &map2);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (forward_wakeup_use_mask) {
|
|
map = idle_cpus_mask;
|
|
CPU_NAND(&map, &dontuse);
|
|
|
|
/* If they are both on, compare and use loop if different. */
|
|
if (forward_wakeup_use_loop) {
|
|
if (CPU_CMP(&map, &map2)) {
|
|
printf("map != map2, loop method preferred\n");
|
|
map = map2;
|
|
}
|
|
}
|
|
} else {
|
|
map = map2;
|
|
}
|
|
|
|
/* If we only allow a specific CPU, then mask off all the others. */
|
|
if (cpunum != NOCPU) {
|
|
KASSERT((cpunum <= mp_maxcpus),("forward_wakeup: bad cpunum."));
|
|
iscpuset = CPU_ISSET(cpunum, &map);
|
|
if (iscpuset == 0)
|
|
CPU_ZERO(&map);
|
|
else
|
|
CPU_SETOF(cpunum, &map);
|
|
}
|
|
if (!CPU_EMPTY(&map)) {
|
|
forward_wakeups_delivered++;
|
|
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
|
|
id = pc->pc_cpuid;
|
|
if (!CPU_ISSET(id, &map))
|
|
continue;
|
|
if (cpu_idle_wakeup(pc->pc_cpuid))
|
|
CPU_CLR(id, &map);
|
|
}
|
|
if (!CPU_EMPTY(&map))
|
|
ipi_selected(map, IPI_AST);
|
|
return (1);
|
|
}
|
|
if (cpunum == NOCPU)
|
|
printf("forward_wakeup: Idle processor not found\n");
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
kick_other_cpu(int pri, int cpuid)
|
|
{
|
|
struct pcpu *pcpu;
|
|
int cpri;
|
|
|
|
pcpu = pcpu_find(cpuid);
|
|
if (CPU_ISSET(cpuid, &idle_cpus_mask)) {
|
|
forward_wakeups_delivered++;
|
|
if (!cpu_idle_wakeup(cpuid))
|
|
ipi_cpu(cpuid, IPI_AST);
|
|
return;
|
|
}
|
|
|
|
cpri = pcpu->pc_curthread->td_priority;
|
|
if (pri >= cpri)
|
|
return;
|
|
|
|
#if defined(IPI_PREEMPTION) && defined(PREEMPTION)
|
|
#if !defined(FULL_PREEMPTION)
|
|
if (pri <= PRI_MAX_ITHD)
|
|
#endif /* ! FULL_PREEMPTION */
|
|
{
|
|
ipi_cpu(cpuid, IPI_PREEMPT);
|
|
return;
|
|
}
|
|
#endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */
|
|
|
|
pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED;
|
|
ipi_cpu(cpuid, IPI_AST);
|
|
return;
|
|
}
|
|
#endif /* SMP */
|
|
|
|
#ifdef SMP
|
|
static int
|
|
sched_pickcpu(struct thread *td)
|
|
{
|
|
int best, cpu;
|
|
|
|
mtx_assert(&sched_lock, MA_OWNED);
|
|
|
|
if (THREAD_CAN_SCHED(td, td->td_lastcpu))
|
|
best = td->td_lastcpu;
|
|
else
|
|
best = NOCPU;
|
|
CPU_FOREACH(cpu) {
|
|
if (!THREAD_CAN_SCHED(td, cpu))
|
|
continue;
|
|
|
|
if (best == NOCPU)
|
|
best = cpu;
|
|
else if (runq_length[cpu] < runq_length[best])
|
|
best = cpu;
|
|
}
|
|
KASSERT(best != NOCPU, ("no valid CPUs"));
|
|
|
|
return (best);
|
|
}
|
|
#endif
|
|
|
|
void
|
|
sched_add(struct thread *td, int flags)
|
|
#ifdef SMP
|
|
{
|
|
cpuset_t tidlemsk;
|
|
struct td_sched *ts;
|
|
u_int cpu, cpuid;
|
|
int forwarded = 0;
|
|
int single_cpu = 0;
|
|
|
|
ts = td->td_sched;
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
KASSERT((td->td_inhibitors == 0),
|
|
("sched_add: trying to run inhibited thread"));
|
|
KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
|
|
("sched_add: bad thread state"));
|
|
KASSERT(td->td_flags & TDF_INMEM,
|
|
("sched_add: thread swapped out"));
|
|
|
|
KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",
|
|
"prio:%d", td->td_priority, KTR_ATTR_LINKED,
|
|
sched_tdname(curthread));
|
|
KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup",
|
|
KTR_ATTR_LINKED, sched_tdname(td));
|
|
SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
|
|
flags & SRQ_PREEMPTED);
|
|
|
|
|
|
/*
|
|
* Now that the thread is moving to the run-queue, set the lock
|
|
* to the scheduler's lock.
|
|
*/
|
|
if (td->td_lock != &sched_lock) {
|
|
mtx_lock_spin(&sched_lock);
|
|
thread_lock_set(td, &sched_lock);
|
|
}
|
|
TD_SET_RUNQ(td);
|
|
|
|
/*
|
|
* If SMP is started and the thread is pinned or otherwise limited to
|
|
* a specific set of CPUs, queue the thread to a per-CPU run queue.
|
|
* Otherwise, queue the thread to the global run queue.
|
|
*
|
|
* If SMP has not yet been started we must use the global run queue
|
|
* as per-CPU state may not be initialized yet and we may crash if we
|
|
* try to access the per-CPU run queues.
|
|
*/
|
|
if (smp_started && (td->td_pinned != 0 || td->td_flags & TDF_BOUND ||
|
|
ts->ts_flags & TSF_AFFINITY)) {
|
|
if (td->td_pinned != 0)
|
|
cpu = td->td_lastcpu;
|
|
else if (td->td_flags & TDF_BOUND) {
|
|
/* Find CPU from bound runq. */
|
|
KASSERT(SKE_RUNQ_PCPU(ts),
|
|
("sched_add: bound td_sched not on cpu runq"));
|
|
cpu = ts->ts_runq - &runq_pcpu[0];
|
|
} else
|
|
/* Find a valid CPU for our cpuset */
|
|
cpu = sched_pickcpu(td);
|
|
ts->ts_runq = &runq_pcpu[cpu];
|
|
single_cpu = 1;
|
|
CTR3(KTR_RUNQ,
|
|
"sched_add: Put td_sched:%p(td:%p) on cpu%d runq", ts, td,
|
|
cpu);
|
|
} else {
|
|
CTR2(KTR_RUNQ,
|
|
"sched_add: adding td_sched:%p (td:%p) to gbl runq", ts,
|
|
td);
|
|
cpu = NOCPU;
|
|
ts->ts_runq = &runq;
|
|
}
|
|
|
|
cpuid = PCPU_GET(cpuid);
|
|
if (single_cpu && cpu != cpuid) {
|
|
kick_other_cpu(td->td_priority, cpu);
|
|
} else {
|
|
if (!single_cpu) {
|
|
tidlemsk = idle_cpus_mask;
|
|
CPU_NAND(&tidlemsk, &hlt_cpus_mask);
|
|
CPU_CLR(cpuid, &tidlemsk);
|
|
|
|
if (!CPU_ISSET(cpuid, &idle_cpus_mask) &&
|
|
((flags & SRQ_INTR) == 0) &&
|
|
!CPU_EMPTY(&tidlemsk))
|
|
forwarded = forward_wakeup(cpu);
|
|
}
|
|
|
|
if (!forwarded) {
|
|
if ((flags & SRQ_YIELDING) == 0 && maybe_preempt(td))
|
|
return;
|
|
else
|
|
maybe_resched(td);
|
|
}
|
|
}
|
|
|
|
if ((td->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_add();
|
|
runq_add(ts->ts_runq, td, flags);
|
|
if (cpu != NOCPU)
|
|
runq_length[cpu]++;
|
|
}
|
|
#else /* SMP */
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
ts = td->td_sched;
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
KASSERT((td->td_inhibitors == 0),
|
|
("sched_add: trying to run inhibited thread"));
|
|
KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
|
|
("sched_add: bad thread state"));
|
|
KASSERT(td->td_flags & TDF_INMEM,
|
|
("sched_add: thread swapped out"));
|
|
KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",
|
|
"prio:%d", td->td_priority, KTR_ATTR_LINKED,
|
|
sched_tdname(curthread));
|
|
KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup",
|
|
KTR_ATTR_LINKED, sched_tdname(td));
|
|
SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
|
|
flags & SRQ_PREEMPTED);
|
|
|
|
/*
|
|
* Now that the thread is moving to the run-queue, set the lock
|
|
* to the scheduler's lock.
|
|
*/
|
|
if (td->td_lock != &sched_lock) {
|
|
mtx_lock_spin(&sched_lock);
|
|
thread_lock_set(td, &sched_lock);
|
|
}
|
|
TD_SET_RUNQ(td);
|
|
CTR2(KTR_RUNQ, "sched_add: adding td_sched:%p (td:%p) to runq", ts, td);
|
|
ts->ts_runq = &runq;
|
|
|
|
/*
|
|
* If we are yielding (on the way out anyhow) or the thread
|
|
* being saved is US, then don't try be smart about preemption
|
|
* or kicking off another CPU as it won't help and may hinder.
|
|
* In the YIEDLING case, we are about to run whoever is being
|
|
* put in the queue anyhow, and in the OURSELF case, we are
|
|
* puting ourself on the run queue which also only happens
|
|
* when we are about to yield.
|
|
*/
|
|
if ((flags & SRQ_YIELDING) == 0) {
|
|
if (maybe_preempt(td))
|
|
return;
|
|
}
|
|
if ((td->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_add();
|
|
runq_add(ts->ts_runq, td, flags);
|
|
maybe_resched(td);
|
|
}
|
|
#endif /* SMP */
|
|
|
|
void
|
|
sched_rem(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
ts = td->td_sched;
|
|
KASSERT(td->td_flags & TDF_INMEM,
|
|
("sched_rem: thread swapped out"));
|
|
KASSERT(TD_ON_RUNQ(td),
|
|
("sched_rem: thread not on run queue"));
|
|
mtx_assert(&sched_lock, MA_OWNED);
|
|
KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq rem",
|
|
"prio:%d", td->td_priority, KTR_ATTR_LINKED,
|
|
sched_tdname(curthread));
|
|
SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);
|
|
|
|
if ((td->td_flags & TDF_NOLOAD) == 0)
|
|
sched_load_rem();
|
|
#ifdef SMP
|
|
if (ts->ts_runq != &runq)
|
|
runq_length[ts->ts_runq - runq_pcpu]--;
|
|
#endif
|
|
runq_remove(ts->ts_runq, td);
|
|
TD_SET_CAN_RUN(td);
|
|
}
|
|
|
|
/*
|
|
* Select threads to run. Note that running threads still consume a
|
|
* slot.
|
|
*/
|
|
struct thread *
|
|
sched_choose(void)
|
|
{
|
|
struct thread *td;
|
|
struct runq *rq;
|
|
|
|
mtx_assert(&sched_lock, MA_OWNED);
|
|
#ifdef SMP
|
|
struct thread *tdcpu;
|
|
|
|
rq = &runq;
|
|
td = runq_choose_fuzz(&runq, runq_fuzz);
|
|
tdcpu = runq_choose(&runq_pcpu[PCPU_GET(cpuid)]);
|
|
|
|
if (td == NULL ||
|
|
(tdcpu != NULL &&
|
|
tdcpu->td_priority < td->td_priority)) {
|
|
CTR2(KTR_RUNQ, "choosing td %p from pcpu runq %d", tdcpu,
|
|
PCPU_GET(cpuid));
|
|
td = tdcpu;
|
|
rq = &runq_pcpu[PCPU_GET(cpuid)];
|
|
} else {
|
|
CTR1(KTR_RUNQ, "choosing td_sched %p from main runq", td);
|
|
}
|
|
|
|
#else
|
|
rq = &runq;
|
|
td = runq_choose(&runq);
|
|
#endif
|
|
|
|
if (td) {
|
|
#ifdef SMP
|
|
if (td == tdcpu)
|
|
runq_length[PCPU_GET(cpuid)]--;
|
|
#endif
|
|
runq_remove(rq, td);
|
|
td->td_flags |= TDF_DIDRUN;
|
|
|
|
KASSERT(td->td_flags & TDF_INMEM,
|
|
("sched_choose: thread swapped out"));
|
|
return (td);
|
|
}
|
|
return (PCPU_GET(idlethread));
|
|
}
|
|
|
|
void
|
|
sched_preempt(struct thread *td)
|
|
{
|
|
|
|
SDT_PROBE2(sched, , , surrender, td, td->td_proc);
|
|
thread_lock(td);
|
|
if (td->td_critnest > 1)
|
|
td->td_owepreempt = 1;
|
|
else
|
|
mi_switch(SW_INVOL | SW_PREEMPT | SWT_PREEMPT, NULL);
|
|
thread_unlock(td);
|
|
}
|
|
|
|
void
|
|
sched_userret(struct thread *td)
|
|
{
|
|
/*
|
|
* XXX we cheat slightly on the locking here to avoid locking in
|
|
* the usual case. Setting td_priority here is essentially an
|
|
* incomplete workaround for not setting it properly elsewhere.
|
|
* Now that some interrupt handlers are threads, not setting it
|
|
* properly elsewhere can clobber it in the window between setting
|
|
* it here and returning to user mode, so don't waste time setting
|
|
* it perfectly here.
|
|
*/
|
|
KASSERT((td->td_flags & TDF_BORROWING) == 0,
|
|
("thread with borrowed priority returning to userland"));
|
|
if (td->td_priority != td->td_user_pri) {
|
|
thread_lock(td);
|
|
td->td_priority = td->td_user_pri;
|
|
td->td_base_pri = td->td_user_pri;
|
|
thread_unlock(td);
|
|
}
|
|
}
|
|
|
|
void
|
|
sched_bind(struct thread *td, int cpu)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED|MA_NOTRECURSED);
|
|
KASSERT(td == curthread, ("sched_bind: can only bind curthread"));
|
|
|
|
ts = td->td_sched;
|
|
|
|
td->td_flags |= TDF_BOUND;
|
|
#ifdef SMP
|
|
ts->ts_runq = &runq_pcpu[cpu];
|
|
if (PCPU_GET(cpuid) == cpu)
|
|
return;
|
|
|
|
mi_switch(SW_VOL, NULL);
|
|
#endif
|
|
}
|
|
|
|
void
|
|
sched_unbind(struct thread* td)
|
|
{
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
KASSERT(td == curthread, ("sched_unbind: can only bind curthread"));
|
|
td->td_flags &= ~TDF_BOUND;
|
|
}
|
|
|
|
int
|
|
sched_is_bound(struct thread *td)
|
|
{
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
return (td->td_flags & TDF_BOUND);
|
|
}
|
|
|
|
void
|
|
sched_relinquish(struct thread *td)
|
|
{
|
|
thread_lock(td);
|
|
mi_switch(SW_VOL | SWT_RELINQUISH, NULL);
|
|
thread_unlock(td);
|
|
}
|
|
|
|
int
|
|
sched_load(void)
|
|
{
|
|
return (sched_tdcnt);
|
|
}
|
|
|
|
int
|
|
sched_sizeof_proc(void)
|
|
{
|
|
return (sizeof(struct proc));
|
|
}
|
|
|
|
int
|
|
sched_sizeof_thread(void)
|
|
{
|
|
return (sizeof(struct thread) + sizeof(struct td_sched));
|
|
}
|
|
|
|
fixpt_t
|
|
sched_pctcpu(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
ts = td->td_sched;
|
|
return (ts->ts_pctcpu);
|
|
}
|
|
|
|
#ifdef RACCT
|
|
/*
|
|
* Calculates the contribution to the thread cpu usage for the latest
|
|
* (unfinished) second.
|
|
*/
|
|
fixpt_t
|
|
sched_pctcpu_delta(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
fixpt_t delta;
|
|
int realstathz;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
ts = td->td_sched;
|
|
delta = 0;
|
|
realstathz = stathz ? stathz : hz;
|
|
if (ts->ts_cpticks != 0) {
|
|
#if (FSHIFT >= CCPU_SHIFT)
|
|
delta = (realstathz == 100)
|
|
? ((fixpt_t) ts->ts_cpticks) <<
|
|
(FSHIFT - CCPU_SHIFT) :
|
|
100 * (((fixpt_t) ts->ts_cpticks)
|
|
<< (FSHIFT - CCPU_SHIFT)) / realstathz;
|
|
#else
|
|
delta = ((FSCALE - ccpu) *
|
|
(ts->ts_cpticks *
|
|
FSCALE / realstathz)) >> FSHIFT;
|
|
#endif
|
|
}
|
|
|
|
return (delta);
|
|
}
|
|
#endif
|
|
|
|
void
|
|
sched_tick(int cnt)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* The actual idle process.
|
|
*/
|
|
void
|
|
sched_idletd(void *dummy)
|
|
{
|
|
struct pcpuidlestat *stat;
|
|
|
|
THREAD_NO_SLEEPING();
|
|
stat = DPCPU_PTR(idlestat);
|
|
for (;;) {
|
|
mtx_assert(&Giant, MA_NOTOWNED);
|
|
|
|
while (sched_runnable() == 0) {
|
|
cpu_idle(stat->idlecalls + stat->oldidlecalls > 64);
|
|
stat->idlecalls++;
|
|
}
|
|
|
|
mtx_lock_spin(&sched_lock);
|
|
mi_switch(SW_VOL | SWT_IDLE, NULL);
|
|
mtx_unlock_spin(&sched_lock);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* A CPU is entering for the first time or a thread is exiting.
|
|
*/
|
|
void
|
|
sched_throw(struct thread *td)
|
|
{
|
|
/*
|
|
* Correct spinlock nesting. The idle thread context that we are
|
|
* borrowing was created so that it would start out with a single
|
|
* spin lock (sched_lock) held in fork_trampoline(). Since we've
|
|
* explicitly acquired locks in this function, the nesting count
|
|
* is now 2 rather than 1. Since we are nested, calling
|
|
* spinlock_exit() will simply adjust the counts without allowing
|
|
* spin lock using code to interrupt us.
|
|
*/
|
|
if (td == NULL) {
|
|
mtx_lock_spin(&sched_lock);
|
|
spinlock_exit();
|
|
PCPU_SET(switchtime, cpu_ticks());
|
|
PCPU_SET(switchticks, ticks);
|
|
} else {
|
|
lock_profile_release_lock(&sched_lock.lock_object);
|
|
MPASS(td->td_lock == &sched_lock);
|
|
td->td_lastcpu = td->td_oncpu;
|
|
td->td_oncpu = NOCPU;
|
|
}
|
|
mtx_assert(&sched_lock, MA_OWNED);
|
|
KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
|
|
cpu_throw(td, choosethread()); /* doesn't return */
|
|
}
|
|
|
|
void
|
|
sched_fork_exit(struct thread *td)
|
|
{
|
|
|
|
/*
|
|
* Finish setting up thread glue so that it begins execution in a
|
|
* non-nested critical section with sched_lock held but not recursed.
|
|
*/
|
|
td->td_oncpu = PCPU_GET(cpuid);
|
|
sched_lock.mtx_lock = (uintptr_t)td;
|
|
lock_profile_obtain_lock_success(&sched_lock.lock_object,
|
|
0, 0, __FILE__, __LINE__);
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
|
|
}
|
|
|
|
char *
|
|
sched_tdname(struct thread *td)
|
|
{
|
|
#ifdef KTR
|
|
struct td_sched *ts;
|
|
|
|
ts = td->td_sched;
|
|
if (ts->ts_name[0] == '\0')
|
|
snprintf(ts->ts_name, sizeof(ts->ts_name),
|
|
"%s tid %d", td->td_name, td->td_tid);
|
|
return (ts->ts_name);
|
|
#else
|
|
return (td->td_name);
|
|
#endif
|
|
}
|
|
|
|
#ifdef KTR
|
|
void
|
|
sched_clear_tdname(struct thread *td)
|
|
{
|
|
struct td_sched *ts;
|
|
|
|
ts = td->td_sched;
|
|
ts->ts_name[0] = '\0';
|
|
}
|
|
#endif
|
|
|
|
void
|
|
sched_affinity(struct thread *td)
|
|
{
|
|
#ifdef SMP
|
|
struct td_sched *ts;
|
|
int cpu;
|
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
|
|
|
/*
|
|
* Set the TSF_AFFINITY flag if there is at least one CPU this
|
|
* thread can't run on.
|
|
*/
|
|
ts = td->td_sched;
|
|
ts->ts_flags &= ~TSF_AFFINITY;
|
|
CPU_FOREACH(cpu) {
|
|
if (!THREAD_CAN_SCHED(td, cpu)) {
|
|
ts->ts_flags |= TSF_AFFINITY;
|
|
break;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If this thread can run on all CPUs, nothing else to do.
|
|
*/
|
|
if (!(ts->ts_flags & TSF_AFFINITY))
|
|
return;
|
|
|
|
/* Pinned threads and bound threads should be left alone. */
|
|
if (td->td_pinned != 0 || td->td_flags & TDF_BOUND)
|
|
return;
|
|
|
|
switch (td->td_state) {
|
|
case TDS_RUNQ:
|
|
/*
|
|
* If we are on a per-CPU runqueue that is in the set,
|
|
* then nothing needs to be done.
|
|
*/
|
|
if (ts->ts_runq != &runq &&
|
|
THREAD_CAN_SCHED(td, ts->ts_runq - runq_pcpu))
|
|
return;
|
|
|
|
/* Put this thread on a valid per-CPU runqueue. */
|
|
sched_rem(td);
|
|
sched_add(td, SRQ_BORING);
|
|
break;
|
|
case TDS_RUNNING:
|
|
/*
|
|
* See if our current CPU is in the set. If not, force a
|
|
* context switch.
|
|
*/
|
|
if (THREAD_CAN_SCHED(td, td->td_oncpu))
|
|
return;
|
|
|
|
td->td_flags |= TDF_NEEDRESCHED;
|
|
if (td != curthread)
|
|
ipi_cpu(cpu, IPI_AST);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
#endif
|
|
}
|